framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,1,0.03389333436886469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,1,0.025914666553338368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,3,0.03450666616360346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,3,0.027034667630990345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,31,0.03942933430274328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,63,0.035274667044480644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,127,0.05958400170008341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,127,0.04372799893220266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,255,0.09073600172996521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,7,0.03514133393764496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,15,0.03880000114440918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,15,0.03524799893299738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,7,0.02792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,63,0.04358399907747904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,31,0.03513066718975703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,255,0.06658133367697398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,511,0.1890933314959208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,1023,0.37540264924367267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,1023,0.19851734240849814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,511,0.10870400071144104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,2047,0.7299573421478271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,4095,0.7412106990814209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,float16,4095,1.4446454048156738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,1,0.00843733362853527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,1,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,64,128,1,float16,fp8,2047,0.38074668248494464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,3,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,7,0.00843733362853527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,7,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,3,0.009098666409651438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,15,0.008746666833758354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,15,0.009818666925032934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,31,0.009328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,63,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,63,0.012416000167528788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,255,0.0107893335322539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,127,0.01251199965675672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,255,0.018645333747069042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,511,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,511,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,127,0.010106666634480158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,1023,0.02980799973011017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,1023,0.02961066613594691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,2047,0.038719999293486275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,4095,0.05225066840648651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,2047,0.03710933278004328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,4095,0.06434666613737743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,8191,0.09109333157539368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,8191,0.12061867117881775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,float16,16383,0.2061226765314738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,64,128,1,float16,fp8,16383,0.16085333625475565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,3,0.00867733359336853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,1,0.010213333492477735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,7,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,3,0.011877333124478659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,15,0.009818666925032934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,7,0.010399999717871347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,31,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,15,0.012175999581813812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,31,0.012005332857370377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,63,0.012527999778588613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,63,0.009882666791478792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,127,0.009797333429257074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,127,0.01251199965675672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,255,0.01099733387430509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,511,0.02870933214823405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,255,0.012618667135636011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,511,0.028346667687098186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,1023,0.03379199902216593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,1023,0.03271466741959254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,2047,0.05478399991989136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,4095,0.10915199915568034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,4095,0.07329066594441731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,2047,0.04470933477083842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,8191,0.19348265727361044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,8191,0.1334826648235321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,fp8,16383,0.2421919902165731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,64,128,1,float16,float16,16383,0.35990933577219647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,1,0.06074133515357971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,1,0.04789333542188009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,3,0.06211199859778086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,3,0.049728001157442726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,15,0.07226666808128357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,31,0.07865599791208903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,7,0.051818668842315674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,7,0.06272533535957336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,31,0.06487466891606648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,15,0.06493333478768666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,255,0.12468799948692322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,255,0.16542933384577432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,127,0.08308800061543782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,63,0.08509866396586101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,511,0.3468639850616455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,127,0.10682666301727295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,63,0.06796266635258992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,511,0.20778665939966837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,1023,0.7098506291707357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,1023,0.38414398829142254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,float16,2047,1.3932959238688152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,1,0.01404800017674764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,1,0.009125333279371262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,64,128,1,float16,fp8,2047,0.7407946586608887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,3,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,3,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,7,0.013957332819700241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,15,0.009872000043590864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,15,0.014544000228246054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,7,0.009482666850090027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,31,0.015722667177518208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,31,0.011834666132926941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,63,0.01632533346613248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,63,0.011930666863918304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,127,0.016384000579516094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,127,0.011994666109482447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,255,0.018810667097568512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,255,0.013669333110253016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,511,0.027552001178264618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,511,0.01916266605257988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,1023,0.02945599953333537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,1023,0.05068266888459524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,2047,0.11128532886505127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,2047,0.056890666484832764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,4095,0.20609599351882935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,4095,0.10433600346247356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,8191,0.3959360122680664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,8191,0.19685866435368857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,float16,16383,0.7762826283772787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,64,128,1,float16,fp8,16383,0.38253335158030194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,1,0.11457066734631856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,1,0.09105599919954936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,3,0.11739200353622437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,3,0.0939626693725586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,7,0.1188800036907196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,7,0.09844266374905904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,15,0.14888532956441244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,15,0.12499200304349263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,31,0.15656532843907675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,63,0.13199466466903687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,31,0.13005866607030234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,127,0.2009920080502828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,63,0.15997333327929178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,127,0.15837333599726358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,255,0.3122933308283488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,255,0.24018667141596475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,511,0.6583199898401896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,float16,1023,1.3865493138631184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,511,0.40307732423146564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,1,0.22408533096313477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,64,128,1,float16,fp8,1023,0.7453599770863851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,3,0.23662932713826498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,1,0.17642666896184286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,3,0.183296004931132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,7,0.2563199996948242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,15,0.2945813337961833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,31,0.3055573304494222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,7,0.19803200165430704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,63,0.3116319974263509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,15,0.2544106642405192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,31,0.25521065791447956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,63,0.2569119930267334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,127,0.39004266262054443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,127,0.30795733133951825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,float16,255,0.6030240058898926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,64,128,1,float16,fp8,255,0.4705333312352498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,1,0.4883466561635335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,3,0.5
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,3,0.38969600200653076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,7,0.5070399840672811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,1,0.3752853473027547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,15,0.5812053283055624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,7,0.407914678255717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,15,0.5030613342920939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,31,0.603007992108663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,31,0.5032053391138712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,63,0.6140160163243612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,63,0.5060319900512695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,float16,127,0.7679413159688314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,64,128,1,float16,fp8,127,0.6072800159454346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,1,0.013904000322024027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,1,0.015279999623696009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,3,0.013936000565687815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,7,0.013951999445756277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,3,0.015471999843915304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,7,0.015333333363135656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,15,0.0143306665122509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,15,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,63,0.015103999525308609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,31,0.020319999506076176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,31,0.01443733274936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,63,0.020303999384244282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,255,0.023567999402681988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,127,0.01718933383623759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,255,0.026591998835404713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,127,0.020394666741291683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,511,0.060965334375699363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,511,0.0356480007370313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,1023,0.11205333471298218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,1023,0.05931733548641205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,2047,0.21529599030812582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,2047,0.10643733541170756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,4095,0.20141865809758505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,4095,0.41898131370544434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,8191,0.8304426670074463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,8191,0.3853280146916707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,float16,16383,1.533722718556722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,64,128,1,float16,fp8,16383,0.7535040378570557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,1,0.9727466901143392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,1,0.7534560362497965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,3,0.9918879667917887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,3,0.7718079884847006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,7,1.0065120061238606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,7,0.8105226357777914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,15,1.1553813616434734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,15,0.9988000392913818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,31,1.1994506518046062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,31,0.998522679011027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,float16,63,1.2192853291829426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,64,128,1,float16,fp8,63,1.0042880376180012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,1,1.9415040016174316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,1,1.4995627403259277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,3,1.9776213963826497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,7,2.0050293604532876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,3,1.537402629852295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,7,1.6141972541809082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,15,2.3038613001505532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,1,0.015077333897352219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,1,0.019248000035683315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,3,0.019530666371186573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,3,0.015498666713635126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,15,1.9918452898661296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,float16,31,2.390709400177002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,7,0.0198186660806338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,7,0.015909332782030106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,15,0.021541332205136616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,64,128,1,float16,fp8,31,1.9909173647562664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,15,0.01966399947802226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,31,0.022064000368118286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,31,0.01971199984351794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,63,0.022218666970729828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,63,0.019808000574509304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,127,0.028543998797734577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,127,0.023498666783173878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,255,0.0491893341143926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,255,0.03501333296298981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,511,0.0995306670665741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,511,0.058464000622431435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,1023,0.19342933098475137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,1023,0.1051093339920044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,2047,0.3689813216527303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,2047,0.19934932390848795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,4095,0.7218826611836752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,4095,0.38571735223134357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,float16,8191,1.4291359583536785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,64,128,1,float16,fp8,8191,0.7653280099232992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,1,0.02470933397610982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,1,0.020762667059898376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,3,0.025248001019159954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,3,0.021370666722456615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,7,0.025818665822347004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,7,0.02214933435122172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,15,0.027466667195161183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,31,0.030261332790056866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,31,0.027477333943049114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,15,0.030069333811601002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,63,0.03068266560633977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,63,0.027503999571005504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,127,0.04464533428351084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,127,0.03257599969704946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,255,0.0673173318306605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,255,0.0517439991235733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,511,0.13767466942469278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,511,0.08431999882062276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,1023,0.27477333943049115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,1023,0.1516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,2047,0.29041600227355957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,2047,0.5268533229827881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,float16,4095,1.0349067052205403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,1,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,1,0.00956266683836778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,48,48,128,1,float16,fp8,4095,0.563594659169515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,3,0.008762666955590248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,3,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,7,0.009610666582981745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,7,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,15,0.008965333302815756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,15,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,31,0.010410666465759277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,31,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,63,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,63,0.012351999680201212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,127,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,127,0.012517333030700684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,255,0.01073066641887029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,511,0.017029333859682083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,511,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,1023,0.029461334149042766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,1023,0.02932800104220708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,2047,0.03746666759252548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,2047,0.036677333215872444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,4095,0.051167999704678856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,4095,0.06358399987220764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,8191,0.11058666308720906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,8191,0.08642133076985677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,float16,16383,0.19727466503779092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,1,0.008458666503429413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,48,48,128,1,float16,fp8,16383,0.15661866466204324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,1,0.009914666414260864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,3,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,7,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,7,0.00978133330742518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,15,0.010026666646202406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,15,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,31,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,31,0.010602666685978571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,63,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,127,0.012389333297808966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,63,0.012357333054145178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,255,0.010874666273593903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,255,0.012517333030700684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,127,0.0102186668664217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,511,0.028213332096735638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,511,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,1023,0.03254399945338567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,2047,0.043434664607048035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,2047,0.04930133124192556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,1023,0.03230933348337809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,4095,0.07052266597747803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,4095,0.08985066413879395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,8191,0.16029333074887595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,16383,0.22949866453806558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,float16,16383,0.3064906597137451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,48,48,128,1,float16,fp8,8191,0.12498133381207784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,1,0.04584533472855886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,1,0.03748266647259394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,3,0.046853333711624146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,3,0.03888533264398575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,7,0.04751466711362203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,7,0.04011733333269755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,15,0.05030933519204458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,15,0.05574400226275126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,31,0.05691733459631602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,63,0.06499200065930684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,31,0.050399998823801674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,63,0.050853331883748375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,127,0.08180266618728638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,127,0.06442133088906606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,255,0.12586133678754172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,255,0.09605333209037781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,511,0.2606719930966695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,1023,0.5359146595001221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,1023,0.29011199871699017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,511,0.15877866744995117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,fp8,2047,0.5617280006408691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,48,48,128,1,float16,float16,2047,1.0411252975463867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,1,0.013866666704416275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,1,0.008965333302815756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,3,0.013679999858140945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,3,0.009343999748428663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,7,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,15,0.014218666901191076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,15,0.00980266680320104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,7,0.01379199946920077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,31,0.015546667079130808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,31,0.01173866664369901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,63,0.01613333324591319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,63,0.011994666109482447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,127,0.01617066686352094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,127,0.011877333124478659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,255,0.01838933303952217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,255,0.013647999614477158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,511,0.02478933334350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,511,0.018885333091020584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,1023,0.047354668378829956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,1023,0.029274667302767437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,2047,0.09566932916641235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,2047,0.05352533360322317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,4095,0.09915733337402344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,4095,0.18429332971572876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,8191,0.35197333494822186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,8191,0.18836265802383423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,float16,16383,0.6887946923573812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,48,48,128,1,float16,fp8,16383,0.3646506468454997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,1,0.08763200044631958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,1,0.06945600112279256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,3,0.07211199899514516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,7,0.09078400333722432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,15,0.10843200484911601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,7,0.07507733503977458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,3,0.08950400352478027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,15,0.094458669424057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,31,0.11943466464678447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,31,0.09635200103123982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,63,0.12267733613650005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,63,0.10110933581988017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,127,0.15425599614779154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,127,0.12065066893895467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,255,0.23847466707229614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,255,0.1818880041440328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,511,0.4967199961344401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,511,0.3047039906183879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,fp8,1023,0.5599679946899414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,48,48,128,1,float16,float16,1023,1.049397309621175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,1,0.1695573329925537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,1,0.1341493328412374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,3,0.17351466417312622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,3,0.13874666889508566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,7,0.18819733460744223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,7,0.14618666966756186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,15,0.22323199113210043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,15,0.1898720065752665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,31,0.2315359910329183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,31,0.19303999344507852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,63,0.2359093427658081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,63,0.1946986714998881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,127,0.2956426739692688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,127,0.23322133223215738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,float16,255,0.4594613313674927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,48,48,128,1,float16,fp8,255,0.35496000448862713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,1,0.35304001967112225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,1,0.26686932643254596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,3,0.3774293263753255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,3,0.28672534227371216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,7,0.38333332538604736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,7,0.3086026708285014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,15,0.43887468179066974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,15,0.3795146544774373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,31,0.45510931809743244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,31,0.3795786698659261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,63,0.4636586507161458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,63,0.38159998257954914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,float16,127,0.5797226826349894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,1,0.019472000499566395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,3,0.01970133309563001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,48,48,128,1,float16,fp8,127,0.4578026533126831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,7,0.019541333119074505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,7,0.015306666493415833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,15,0.020047999918460846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,15,0.015082667271296183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,31,0.022122666239738464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,63,0.022672000030676525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,31,0.016938666502634685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,127,0.02312533309062322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,63,0.017258666455745697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,127,0.017024000485738117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,255,0.01971199984351794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,511,0.046069333950678505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,255,0.026522666215896606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,511,0.027141332626342773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,1023,0.07355199754238129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,1023,0.04700266818205515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,2047,0.162773331006368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,2047,0.08356799681981404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,4095,0.30429333448410034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,4095,0.15796266992886862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,8191,0.5890933275222778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,8191,0.3004426757494609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,float16,16383,1.1562026341756184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,48,48,128,1,float16,fp8,16383,0.5881760120391846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,float16,1,0.734112024307251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,fp8,1,0.5682506561279297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,float16,3,0.7471466859181722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,fp8,3,0.5823786656061808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,float16,7,0.7589066823323568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,fp8,7,0.6115946769714355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,float16,15,0.8715840180714926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,float16,31,0.9025279680887858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,fp8,15,0.7522719701131185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,fp8,31,0.7520586649576823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,float16,63,0.9186293284098307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,48,48,128,1,float16,fp8,63,0.755669355392456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,float16,1,1.4611520767211914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,fp8,1,1.1312800248463948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,float16,3,1.4873706499735515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,fp8,3,1.1585546334584553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,fp8,7,1.2154826323191326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,float16,15,1.7359414100646973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,float16,7,1.509109338124593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,fp8,15,1.4975840250651042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,float16,31,1.7988053957621257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,1,0.014197333405415217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,1,0.013946666071812311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,3,0.014335999886194864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,3,0.014511999984582266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,7,0.014794666320085526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,7,0.014442666123310724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,15,0.016986666868130367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,15,0.01621333385507266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,31,0.0169813334941864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,48,48,128,1,float16,fp8,31,1.4962293306986492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,31,0.01632533346613248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,63,0.016149333367745083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,63,0.01720533271630605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,127,0.020010666300853092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,127,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,255,0.036133334040641785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,255,0.026895999908447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,511,0.07212799787521362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,511,0.045647998650868736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,1023,0.14215466380119324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,1023,0.08125866452852885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,2047,0.26969067255655926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,2047,0.15562132994333902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,4095,0.5233920017878214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,4095,0.29873067140579224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,float16,8191,1.036415974299113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,1,0.019248000035683315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,48,48,128,1,float16,fp8,8191,0.5880320072174072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,1,0.023962666591008503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,3,0.019674666225910187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,3,0.024501333634058636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,7,0.02474133421977361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,7,0.019776000330845516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,15,0.026741333305835724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,15,0.02380266785621643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,31,0.027258666853109997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,31,0.023711999257405598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,63,0.027466667195161183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,63,0.023711999257405598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,127,0.039690665900707245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,127,0.027664000789324444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,255,0.06159999966621399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,255,0.0436160018046697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,511,0.1263146698474884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,511,0.0711893339951833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,1023,0.2444266676902771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,1023,0.12804266810417175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,2047,0.4761173327763875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,2047,0.24597332874933878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,fp8,4095,0.4753066698710124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,40,40,128,1,float16,float16,4095,0.9320106506347656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,1,0.008346666892369589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,1,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,3,0.008586666857202848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,3,0.009797333429257074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,7,0.00978133330742518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,7,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,15,0.00897066667675972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,15,0.009839999799927076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,31,0.009482666850090027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,63,0.009493333597977957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,63,0.012250666817029318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,127,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,127,0.012383999923865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,255,0.010666667173306147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,255,0.01240533341964086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,511,0.016837333639462788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,1023,0.018079999834299088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,1023,0.017786666750907898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,511,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,2047,0.032586666444937386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,2047,0.032218667368094124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,4095,0.046821330984433494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,4095,0.0431573341290156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,8191,0.08899199962615967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,float16,16383,0.1528320014476776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,1,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,8191,0.06903466582298279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,40,40,128,1,float16,fp8,16383,0.1236799955368042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,1,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,3,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,3,0.00847999999920527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,7,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,7,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,15,0.008853333070874214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,15,0.009957333405812582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,31,0.009754666437705358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,31,0.01044800008336703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,63,0.009514666472872099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,63,0.01227733368674914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,127,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,127,0.01259200026591619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,255,0.010981333752473196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,255,0.012495999534924826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,511,0.02790933350721995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,511,0.027664000789324444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,1023,0.03241066634654999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,1023,0.032287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,2047,0.04646400113900503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,2047,0.04318933188915253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,4095,0.08826133608818054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,4095,0.06790933509667714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,8191,0.15307199954986572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,float16,16383,0.28066666920979816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,8191,0.12377066413561504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,40,40,128,1,float16,fp8,16383,0.2234506607055664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,1,0.04014399896065394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,1,0.03170666595300039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,3,0.040965333580970764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,3,0.03310933212439219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,7,0.041322665909926094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,7,0.03412266572316488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,15,0.04711466530958811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,15,0.04309333364168803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,31,0.0428959975639979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,31,0.04779199759165446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,63,0.05630399783452352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,63,0.04307200014591217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,127,0.07112533350785573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,127,0.05486933390299479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,255,0.10785599549611409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,255,0.0811466674009959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,511,0.22437334060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,511,0.1341973344484965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,1023,0.45448533693949383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,1023,0.24462399880091348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,float16,2047,0.8853387037913004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,1,0.01351999988158544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,40,40,128,1,float16,fp8,2047,0.4716533422470093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,1,0.016165333489576977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,3,0.013674666484196981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,3,0.01599466676513354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,7,0.013733333597580591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,7,0.016037333756685257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,15,0.014074667046467463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,15,0.01648533344268799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,31,0.015599999576807022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,31,0.017583999782800674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,63,0.01584533353646596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,63,0.02143999934196472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,127,0.016010666886965435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,127,0.021477334201335907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,255,0.018181333939234417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,255,0.02147199958562851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,511,0.0242399995525678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,511,0.02470933397610982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,1023,0.04553600152333578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,1023,0.0349440003434817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,2047,0.09403733412424724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,2047,0.05739733576774597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,4095,0.17489065726598105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,4095,0.104912002881368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,8191,0.3434986670811971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,8191,0.19260267416636148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,float16,16383,0.6722666422526041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,40,40,128,1,float16,fp8,16383,0.36529068152109784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,1,0.07301333546638489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,3,0.07452799876530965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,1,0.058687999844551086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,3,0.06098666787147522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,7,0.07584533095359802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,7,0.06316266457239787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,15,0.08924800157546997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,15,0.07971733311812083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,31,0.10043733318646748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,31,0.07986666758855183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,63,0.1032960017522176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,63,0.08515200018882751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,127,0.1293653349081675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,127,0.10143466790517171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,255,0.20006400346755981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,255,0.15286399920781454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,511,0.4174240032831828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,511,0.2547733386357625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,float16,1023,0.8742559750874838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,40,40,128,1,float16,fp8,1023,0.468176007270813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,1,0.14191466569900513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,1,0.11263466874758403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,3,0.14537599682807922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,3,0.1167093316713969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,7,0.15056000153223673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,7,0.12260799606641133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,15,0.18743467330932617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,15,0.1574026644229889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,31,0.194922665754954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,31,0.1620693306128184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,63,0.19814932346343994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,63,0.16336533427238464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,127,0.24834134181340536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,float16,255,0.38526399930318195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,127,0.19590934117635092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,40,40,128,1,float16,fp8,255,0.2974933385848999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,1,0.2831839919090271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,1,0.22105065981547037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,3,0.3111093242963155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,3,0.2323360045750936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,7,0.25633599360783893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,15,0.36849065621693927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,7,0.3205813368161519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,15,0.317466676235199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,31,0.3816800117492676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,63,0.38836801052093506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,1,0.01905599981546402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,31,0.31783467531204224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,1,0.014826666563749313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,63,0.3192426760991414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,fp8,127,0.38306665420532227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,40,40,128,1,float16,float16,127,0.485365351041158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,3,0.019098666807015736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,3,0.014058666924635569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,7,0.015008000036080679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,7,0.019167999426523846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,15,0.019941333681344986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,15,0.014762666076421738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,31,0.021829334398110706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,63,0.02233600119749705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,31,0.01676799977819125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,127,0.02254933367172877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,127,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,63,0.0169813334941864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,255,0.019482667247454327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,255,0.025888000925381977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,511,0.043951998154322304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,1023,0.07191466788450877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,2047,0.08251200119654338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,1023,0.045696000258127846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,2047,0.14920000235239664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,511,0.026863999664783478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,4095,0.28412799040476483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,4095,0.15152532855669656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,8191,0.547871987024943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,8191,0.2909653385480245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,float16,16383,1.0781866709391277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,40,40,128,1,float16,fp8,16383,0.5671306848526001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,float16,1,0.6128746668497721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,fp8,1,0.4755680163701375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,float16,3,0.6267040173212687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,fp8,3,0.48744531472524005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,float16,7,0.6350346803665161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,fp8,7,0.5114666620890299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,float16,15,0.7293386459350586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,fp8,15,0.6278613408406576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,float16,31,0.7554240226745605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,fp8,31,0.6282773415247599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,float16,63,0.7682773272196451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,40,40,128,1,float16,fp8,63,0.6310826539993286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,float16,1,1.2203466892242432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,fp8,1,0.9441119829813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,float16,3,1.2453493277231853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,float16,7,1.2621013323465984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,fp8,3,0.9662986596425375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,fp8,7,1.0156906445821126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,float16,15,1.4531680742899578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,1,0.013872000078360239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,1,0.013712000101804733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,fp8,15,1.249626636505127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,float16,31,1.5037226676940918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,3,0.014202666779359182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,3,0.013642666240533194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,40,40,128,1,float16,fp8,31,1.2492266496022542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,7,0.014335999886194864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,7,0.013722666849692663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,15,0.015482666591803232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,15,0.015263999501864115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,31,0.015626666446526844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,31,0.015040000279744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,63,0.01674666628241539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,63,0.014933332800865173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,127,0.018138666947682697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,127,0.01695999999841054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,255,0.032074667513370514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,255,0.023578666150569916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,511,0.06558399895826976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,511,0.04055999964475632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,1023,0.12705066800117493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,1023,0.07208533088366191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,2047,0.24034132560094199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,2047,0.13639466961224875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,4095,0.46535468101501465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,4095,0.2659893234570821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,float16,8191,0.9185333251953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,40,40,128,1,float16,fp8,8191,0.5226346651713053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,1,0.019141333798567455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,3,0.019573333362738293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,3,0.015471999843915304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,7,0.01998399943113327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,7,0.01590399940808614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,15,0.021802666286627453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,15,0.01972266659140587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,31,0.021935999393463135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,31,0.01978133370478948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,63,0.02214933435122172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,63,0.019866666446129482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,127,0.027749332288901012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,127,0.023317334552605946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,255,0.049173335234324135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,255,0.03486400097608566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,511,0.10199999809265137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,1023,0.1939093271891276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,511,0.058559998869895935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,1023,0.1048906644185384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,2047,0.37026135126749676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,2047,0.1992266575495402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,4095,0.7224160035451254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,4095,0.3858240048090617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,float16,8191,1.4290986061096191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,32,128,1,float16,fp8,8191,0.7655413150787354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,1,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,1,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,3,0.008613333106040955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,3,0.00978133330742518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,7,0.008698666468262672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,7,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,15,0.009813333551088968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,31,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,15,0.008693333094318708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,31,0.010389333590865135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,63,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,63,0.012442667037248611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,127,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,127,0.012213333199421564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,255,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,255,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,511,0.01676799977819125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,511,0.01676799977819125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,1023,0.01801066721479098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,1023,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,2047,0.029824001093705494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,2047,0.029391999046007793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,4095,0.04089066634575526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,4095,0.0397173340121905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,8191,0.07784000039100647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,8191,0.06015466650327047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,16383,0.12950399518013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,32767,0.18229333559672037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,fp8,16383,0.10383466879526775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,32,128,1,float16,float16,32767,0.232640008131663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,1,0.008394666636983553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,1,0.009797333429257074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,3,0.008623999853928884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,3,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,7,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,15,0.008725333337982496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,7,0.008416000132759413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,15,0.009925333162148794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,31,0.009493333597977957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,31,0.01073066641887029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,63,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,127,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,63,0.012272000312805176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,127,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,255,0.010682666053374609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,255,0.01232533281048139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,511,0.017136000096797943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,511,0.016821333517630894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,1023,0.029802667597929638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,1023,0.029530666768550873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,2047,0.03879466652870178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,2047,0.03712533414363861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,4095,0.06688533226648967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,4095,0.05250666538874308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,8191,0.11598400274912517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,8191,0.0888159970442454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,16383,0.2074399987856547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,16383,0.16053332885106406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,1,0.034485332667827606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,fp8,32767,0.29660266637802124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,32,128,1,float16,float16,32767,0.3877280155817668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,1,0.02640533447265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,3,0.03501333296298981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,3,0.027210667729377747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,7,0.03532800078392029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,7,0.02804800122976303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,15,0.03924266745646795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,15,0.035445332527160645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,31,0.03957333415746689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,31,0.035216001172860466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,63,0.0431573341290156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,63,0.03534399966398875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,127,0.06009600063165029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,127,0.043280000487963356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,255,0.09053867061932881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,255,0.06658666829268138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,511,0.18978132804234824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,511,0.10930666327476501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,1023,0.19833600521087646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,1023,0.37569065888722736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,2047,0.7281386852264404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,2047,0.38060800234476727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,float16,4095,1.4442933400472004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,32,128,1,float16,fp8,4095,0.7413439750671387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,1,0.00877333308259646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,1,0.009957333405812582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,3,0.008746666833758354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,3,0.009946666657924652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,7,0.008874666566650072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,7,0.0099093330403169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,15,0.008943999807039896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,15,0.009925333162148794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,31,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,31,0.010698666175206503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,63,0.009930666536092758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,63,0.01239466667175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,127,0.009882666791478792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,127,0.01239466667175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,255,0.011061333119869232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,255,0.012698666503032049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,511,0.028277332584063213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,511,0.02865600089232127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,1023,0.03379199902216593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,1023,0.03297599901755651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,2047,0.0544106662273407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,2047,0.044693330923716225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,8191,0.19146132469177246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,4095,0.0730506678422292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,4095,0.108815997838974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,8191,0.1334773302078247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,16383,0.35973866780598956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,16383,0.24236800273259482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,float16,32767,0.6938133239746094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,1,0.06086933116118113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,1,0.04804266492525736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,32,128,1,float16,fp8,32767,0.45900265375773114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,3,0.0622026671965917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,3,0.04986133178075155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,7,0.06273066500822704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,7,0.05198933184146881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,15,0.07252799967924754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,15,0.06530666848023732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,31,0.07891199986139934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,31,0.06515199939409892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,63,0.08547733227411906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,63,0.0678773323694865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,127,0.1077280044555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,127,0.083146666487058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,255,0.16555200020472208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,255,0.12495999534924825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,511,0.3432053327560425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,511,0.20839466651280722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,1023,0.7102453708648682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,1023,0.38359467188517254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,float16,2047,1.3963732719421387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,32,128,1,float16,fp8,2047,0.7411306699117025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,1,0.1143999993801117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,1,0.09130666653315227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,3,0.11737599968910217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,3,0.09428266684214275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,7,0.11911466717720032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,7,0.09923733274141948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,15,0.1498240033785502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,31,0.15734933813412985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,15,0.12530666589736938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,31,0.13024533788363138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,63,0.1606826682885488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,63,0.13246933619181314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,127,0.20057066281636557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,127,0.15854400396347046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,255,0.31301865975062054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,255,0.2402026653289795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,float16,511,0.6530933380126953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,32,128,1,float16,fp8,511,0.4037333329518636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,1,0.22405866781870523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,1,0.17726399501164755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,3,0.23878933986028036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,3,0.18367999792099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,7,0.2579519947369893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,7,0.20011200507481894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,15,0.2966559926668803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,15,0.2548159956932068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,31,0.3072959979375203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,31,0.2558293342590332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,63,0.312389334042867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,63,0.2568693359692891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,127,0.39082133769989014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,127,0.3086506724357605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,1,0.014186666657527288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,1,0.009061333412925402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,float16,255,0.6049386660257975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,32,128,1,float16,fp8,255,0.47019731998443604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,3,0.01421333352724711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,3,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,7,0.014186666657527288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,7,0.009429333110650381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,15,0.01461333284775416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,31,0.01598400001724561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,15,0.009973333527644476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,31,0.011946666985750198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,63,0.016362667083740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,63,0.011813333878914515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,127,0.016565332810084026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,255,0.013525333255529404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,127,0.0120319997270902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,255,0.018901333212852478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,511,0.019071999937295914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,511,0.027237333357334137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,2047,0.11107200384140015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,1023,0.050981332858403526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,1023,0.02991466720898946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,2047,0.05637866755326589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,4095,0.20551466941833496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,4095,0.10426666339238484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,8191,0.1967680056889852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,8191,0.396229346593221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,16383,0.774229367574056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,16383,0.38199468453725177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,float16,32767,1.5364480018615723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,32,128,1,float16,fp8,32767,0.9247786998748779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,1,0.49172266324361164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,1,0.3765920003255208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,3,0.5036799907684326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,3,0.39129066467285156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,7,0.5105280081431071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,7,0.41068800290425617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,15,0.5860373179117838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,15,0.5039786497751871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,31,0.504256010055542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,31,0.6075520118077596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,63,0.616869330406189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,63,0.5067093372344971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,float16,127,0.7701919873555502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,32,128,1,float16,fp8,127,0.6081653436024984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,1,0.9792746702829996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,1,0.7579840024312338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,3,0.7764906883239746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,7,1.0133919715881348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,7,0.8145173390706381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,3,0.9992799758911133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,15,1.1657226880391438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,1,0.01360000049074491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,1,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,31,1.207642634709676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,15,1.0009973049163818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,3,0.013701333353916803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,31,1.0010826587677002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,float16,63,1.225279966990153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,15,0.01452800010641416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,32,128,1,float16,fp8,63,1.0046079953511555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,3,0.015285332997639975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,7,0.013872000078360239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,7,0.015573333948850632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,15,0.01672533278663953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,31,0.014794666320085526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,31,0.020175999651352566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,63,0.015173333386580149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,63,0.02021866664290428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,127,0.017349333812793095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,511,0.06421866516272227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,127,0.020341333001852036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,255,0.023658665517965954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,255,0.02625600000222524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,2047,0.21797333161036173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,511,0.03544000039498011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,1023,0.116949329773585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,1023,0.059605335195859276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,2047,0.10735999544461568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,8191,0.8209280172983805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,4095,0.20085332791010538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,4095,0.4166719913482666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,8191,0.38466131687164307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,1,0.01413333291808764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,1,0.014362666755914688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,fp8,16383,0.753706693649292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,32,128,1,float16,float16,16383,1.5273973147074382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,3,0.01422400027513504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,3,0.014352000008026758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,7,0.01423466702302297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,7,0.014874666929244995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,15,0.01695999999841054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,15,0.01634666696190834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,63,0.016122666498025257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,31,0.017290666699409485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,31,0.01632533346613248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,63,0.01706133286158244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,255,0.026746665438016255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,127,0.020288000504175823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,127,0.018592000007629395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,255,0.03594133257865906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,511,0.07251200079917908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,1023,0.08130133152008057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,1023,0.14190399646759033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,511,0.04584533472855886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,8191,1.0360906918843586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,2047,0.2695680061976115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,2047,0.15627200404802957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,float16,4095,0.5249600013097128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,4095,0.29870400826136273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,1,0.008362666393319765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,24,24,128,1,float16,fp8,8191,0.5898240009943644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,1,0.0129120002190272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,3,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,31,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,7,0.00961599995692571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,3,0.008330666770537695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,7,0.008447999755541483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,15,0.008661333471536636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,63,0.009322666873534521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,31,0.010341333225369453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,63,0.012063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,127,0.012213333199421564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,511,0.01666133354107539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,127,0.009248000259200731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,255,0.010805333654085795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,255,0.012063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,2047,0.018053332964579265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,511,0.016586666305859882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,1023,0.017616000026464462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,1023,0.01749333366751671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,2047,0.01788266624013583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,8191,0.0639519989490509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,4095,0.03752533346414566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,4095,0.03677333394686381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,8191,0.050986667474110924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,16383,0.10963732997576396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,32767,0.15660267074902853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,fp8,16383,0.09057066837946574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,1,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,24,24,128,1,float16,float16,32767,0.19687465826670328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,1,0.00860799973209699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,7,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,3,0.009066666786869368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,3,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,31,0.010373333469033241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,7,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,15,0.008634666601816813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,15,0.009989333028594652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,31,0.00956266683836778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,127,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,63,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,63,0.012309333930412928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,511,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,255,0.0107893335322539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,255,0.012437333663304647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,511,0.016895999511082966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,1023,0.029626667499542236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,2047,0.03676266719897588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,1023,0.029359998802344005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,2047,0.03746666759252548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,4095,0.050901333491007485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,4095,0.06353066861629486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,8191,0.11063466469446818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,16383,0.19709867238998413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,8191,0.08986133337020874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,16383,0.15660799543062845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,3,0.025242666403452556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,float16,32767,0.3714826504389445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,1,0.024693332612514496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,24,24,128,1,float16,fp8,32767,0.28710933526357013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,1,0.020725333442290623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,3,0.02130666623512904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,7,0.021776000658671062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,7,0.025829332570234936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,15,0.030053332448005676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,15,0.027466667195161183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,63,0.02757333219051361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,31,0.027621333797772724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,31,0.030261332790056866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,63,0.030586667358875275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,255,0.05160533388455709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,127,0.044981335600217186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,127,0.032672000428040825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,255,0.06710933148860931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,511,0.1358506679534912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,1023,0.1514079968134562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,511,0.0839359958966573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,1023,0.2748426596323649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,2047,0.5268640120824178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,1,0.008618666479984919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,2047,0.2911626696586609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,float16,4095,1.0354026953379314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,24,24,128,1,float16,fp8,4095,0.5646346807479858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,7,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,3,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,1,0.010373333469033241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,3,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,7,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,15,0.008874666566650072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,31,0.009482666850090027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,15,0.00997866690158844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,31,0.010677333921194077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,255,0.010965333630641302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,63,0.010533332824707031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,63,0.012341332932313284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,127,0.00980266680320104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,127,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,255,0.012538666526476542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,511,0.02792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,511,0.028213332096735638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,4095,0.07070933282375336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,1023,0.032560000816980995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,1023,0.0323786661028862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,2047,0.04994133114814758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,2047,0.04331733286380768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,4095,0.09016533692677815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,8191,0.12500266234079996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,8191,0.15942399700482687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,16383,0.30952000617980957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,1,0.037530665596326195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,float16,32767,0.5618186791737875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,1,0.045968001087506614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,16383,0.2287786602973938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,24,24,128,1,float16,fp8,32767,0.4331573247909546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,3,0.04688533147176107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,7,0.04749333361784617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,3,0.038986665507157646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,15,0.05604266623655955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,63,0.06532266736030579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,7,0.04030400017897288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,31,0.05684266487757365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,15,0.0505973349014918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,31,0.05049066742261251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,63,0.050997331738471985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,127,0.06502933303515117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,127,0.08185600241025288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,255,0.12610666950543722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,255,0.0959999958674113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,511,0.2582026720046997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,1023,0.29073067506154376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,1023,0.5344586769739786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,float16,2047,1.0428586800893147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,511,0.15927466750144958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,3,0.08949866890907288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,24,24,128,1,float16,fp8,2047,0.5625706513722738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,1,0.08745599786440532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,3,0.07234666744867961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,1,0.06976533432801564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,15,0.10851732889811198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,7,0.09079999725023906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,7,0.07528533538182576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,15,0.09502933422724406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,31,0.12001599868138631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,63,0.10095999638239543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,31,0.09666666388511658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,127,0.1548479994138082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,63,0.12245333194732666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,127,0.12076266606648763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,511,0.49425601959228516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,float16,255,0.23929067452748617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,255,0.1823413372039795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,24,24,128,1,float16,fp8,511,0.30435200532277423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,1,0.1697280009587606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,1,0.13422933220863342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,3,0.17421332995096842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,3,0.13893866539001465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,7,0.1881706714630127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,7,0.14660267035166422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,15,0.22407466173171997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,15,0.190720001856486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,31,0.23254932959874472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,63,0.23662400245666504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,31,0.19367466370264688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,63,0.19468265771865845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,127,0.29631465673446655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,127,0.2329066594441732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,1,0.013845333208640417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,float16,255,0.4607306718826294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,24,24,128,1,float16,fp8,255,0.35567466417948407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,1,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,3,0.013834666460752487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,7,0.010346666599313417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,7,0.01394133393963178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,3,0.010128000130256018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,15,0.01441066712141037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,63,0.015834666788578033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,15,0.009786666681369146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,31,0.015749332805474598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,31,0.011674666156371435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,63,0.011962667107582092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,255,0.01841066653529803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,127,0.011999999483426413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,127,0.016143999993801117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,255,0.013642666240533194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,511,0.024869332710901897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,511,0.01887999971707662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,1023,0.04742933313051859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,1023,0.029290666182835896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,2047,0.09673600395520528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,2047,0.0536106675863266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,4095,0.1840160091718038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,4095,0.09959999720255534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,8191,0.3526560068130493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,8191,0.18836800257364908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,16383,0.6891520023345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,16383,0.3640799919764201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,float16,32767,1.3670612970987956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,24,24,128,1,float16,fp8,32767,0.8823786576588949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,1,0.35388799508412677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,1,0.26826133330663043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,3,0.37911466757456463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,3,0.2873706618944804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,7,0.3844746748606364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,7,0.3094666600227356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,15,0.4416533311208089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,15,0.379749337832133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,31,0.45817601680755615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,31,0.3795520067214966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,63,0.38201598326365155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,63,0.4644746780395508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,float16,127,0.5813226699829102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,24,24,128,1,float16,fp8,127,0.45826133092244464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,1,0.7356053193410238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,1,0.5708479881286621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,3,0.751583973566691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,3,0.5845333337783813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,7,0.6141493320465088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,15,0.876693328221639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,7,0.7616213162740072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,15,0.7526880105336508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,1,0.014981333166360855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,31,0.9085760116577148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,1,0.01966933285196622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,31,0.7525493303934733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,float16,63,0.921066681543986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,3,0.015061333775520325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,24,24,128,1,float16,fp8,63,0.7560373147328695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,3,0.019744000087181728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,7,0.01969066634774208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,7,0.015109332899252573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,15,0.020554666717847187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,15,0.015210667004187902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,31,0.0222080002228419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,31,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,63,0.022757334013779957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,63,0.01709866647919019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,127,0.022986667851607006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,255,0.019685332973798115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,127,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,255,0.026789332429567974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,511,0.04647466540336609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,2047,0.16328533490498862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,1023,0.07344000041484833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,1023,0.04678399860858917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,511,0.027274665733178455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,2047,0.08348799745241801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,4095,0.3062079946200053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,8191,0.5909546613693237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,4095,0.15813333789507547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,8191,0.30058666070302326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,float16,16383,1.1564533710479736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,1,0.013893333574136099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,1,0.015354666858911514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,3,0.013893333574136099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,24,24,128,1,float16,fp8,16383,0.5877653360366821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,3,0.01532799998919169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,31,0.014671999961137772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,7,0.015557333827018738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,15,0.014458666245142618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,15,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,63,0.020117333779732387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,31,0.020394666741291683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,63,0.014997333288192749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,127,0.01738133281469345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,511,0.06391466657320659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,127,0.02025066688656807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,255,0.026549334327379864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,255,0.023743999501069386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,2047,0.21594132979710898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,511,0.03519999980926514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,1023,0.11386666695276897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,1023,0.059706668059031166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,2047,0.10706667105356853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,4095,0.43104533354441327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,4095,0.20067733526229858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,8191,0.8264586925506592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,3,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,8191,0.38501866658528644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,float16,16383,1.530794620513916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,1,0.008336000144481659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,16,128,1,float16,fp8,16383,0.7562026977539062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,3,0.008474666625261307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,1,0.009317333499590555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,7,0.008933333059151968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,15,0.008517333616813024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,63,0.01210133358836174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,15,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,7,0.010431999961535135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,31,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,31,0.010506667196750641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,63,0.009237333511312803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,127,0.01209066684047381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,127,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,255,0.010405333091815313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,1023,0.017488000293572743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,511,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,511,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,1023,0.017749333133300144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,4095,0.029792000850041706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,2047,0.01791999985774358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,2047,0.017375999440749485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,8191,0.04124800115823746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,4095,0.029674666623274486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,8191,0.040218666195869446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,16383,0.0613919993241628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,16383,0.08004799981911977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,1,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,32767,0.14195199807484946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,32767,0.11450666189193726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,1,0.008538666491707167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,3,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,float16,65535,0.25697600841522217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,3,0.008592000231146812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,16,128,1,float16,fp8,65535,0.20247467358907065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,15,0.009872000043590864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,7,0.008421333506703377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,7,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,15,0.00871999996403853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,31,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,31,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,63,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,63,0.010640000303586325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,127,0.009445333232482275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,511,0.016501333564519882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,127,0.012304000556468964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,255,0.010773333410422007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,511,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,255,0.012165332833925882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,2047,0.029765332738558452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,1023,0.01812800019979477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,1023,0.017429333180189133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,8191,0.05960000058015188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,2047,0.02943466603755951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,4095,0.03988266736268997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,4095,0.041365332901477814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,8191,0.07725866635640462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,32767,0.2330346703529358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,16383,0.12938666343688965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,16383,0.10433600346247356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,32767,0.18225600322087607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,fp8,65535,0.3470613161722819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,16,128,1,float16,float16,65535,0.43745601177215576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,1,0.019317333896954853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,15,0.02165866643190384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,3,0.01543466622630755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,3,0.019567999988794327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,7,0.019941333681344986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,7,0.016127999871969223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,15,0.019776000330845516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,31,0.021957332889238994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,31,0.01972266659140587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,63,0.022085333863894146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,63,0.01988799994190534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,511,0.10077333450317383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,127,0.028064000109831493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,127,0.023418667415777843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,255,0.04887466629346212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,255,0.034714666505654655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,1023,0.10564800103505452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,511,0.058575997749964394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,1023,0.19435199101765951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,2047,0.3708426554997762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,2047,0.20005865891774496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,4095,0.7225013573964437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,float16,8191,1.429141362508138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,4095,0.3860693375269572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,16,128,1,float16,fp8,8191,0.7668960094451904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,7,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,1,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,1,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,3,0.008656000097592672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,3,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,7,0.00983466642598311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,15,0.009818666925032934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,15,0.008746666833758354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,63,0.009754666437705358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,31,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,31,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,127,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,63,0.012442667037248611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,255,0.012389333297808966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,255,0.010762666662534079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,511,0.01717866708834966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,1023,0.029946667452653248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,2047,0.03711999952793121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,511,0.01682666689157486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,1023,0.029546665648619335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,2047,0.038736000657081604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,8191,0.09203733007113139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,4095,0.06508799890677135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,4095,0.05226133267084757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,8191,0.1151146690050761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,16383,0.20570133129755655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,16383,0.16054399808247885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,32767,0.40223999818166095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,32767,0.2977173328399658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,7,0.03525333354870478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,float16,65535,0.7433226903279623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,1,0.03421866645415624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,16,128,1,float16,fp8,65535,0.5656586488087972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,1,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,3,0.03490666548411051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,3,0.027482666075229645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,7,0.027930667002995808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,15,0.03903999924659729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,15,0.035386666655540466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,127,0.05996266504128774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,31,0.03985599925120672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,63,0.04278933505217234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,31,0.035429333647092186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,63,0.035386666655540466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,255,0.06666133304437001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,127,0.04320000112056732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,255,0.090938667456309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,511,0.1893706719080607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,511,0.10922666390736897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,1023,0.3756320079167684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,1023,0.19900800784428915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,2047,0.7330773671468099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,3,0.04978133241335551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,2047,0.38239999612172443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,1,0.060789331793785095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,float16,4095,1.4454933802286785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,1,0.04810666541258494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,16,128,1,float16,fp8,4095,0.7431466579437256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,3,0.06214933097362518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,7,0.06268799801667531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,7,0.052101333936055504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,15,0.07269333302974701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,15,0.06548800071080525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,63,0.06783466537793477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,63,0.08504000306129456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,31,0.07868800063927968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,31,0.0656160016854604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,255,0.12483200430870056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,127,0.10777067144711812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,127,0.083146666487058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,255,0.16544000307718912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,511,0.34466131528218585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,511,0.20856000979741415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,fp8,1023,0.3853333393732707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,16,128,1,float16,float16,1023,0.711674690246582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,1,0.11449600259462993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,1,0.09129066268603007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,3,0.11685333649317424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,3,0.0946666697661082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,7,0.0990133285522461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,7,0.1192586620648702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,15,0.1507253348827362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,15,0.12550399700800577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,31,0.1575040022532145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,31,0.1302826702594757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,63,0.13212266564369202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,63,0.1604746679464976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,127,0.15813333789507547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,127,0.20173333088556925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,255,0.3130026658376058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,255,0.2403093377749125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,1,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,fp8,511,0.4036320050557454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,16,128,1,float16,float16,511,0.6556479930877686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,1,0.009882666791478792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,3,0.008810666700204214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,15,0.010010666524370512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,3,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,7,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,7,0.009919999788204828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,63,0.010869332899649939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,31,0.009583999713261923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,15,0.010533332824707031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,31,0.010703999549150467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,63,0.012341332932313284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,127,0.009919999788204828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,255,0.011461333682139715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,255,0.012762666990359625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,511,0.028346667687098186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,511,0.028704000016053517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,1023,0.03385066737731298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,1023,0.03286933402220408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,2047,0.04496000210444132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,2047,0.053642665346463524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,4095,0.10827199618021648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,8191,0.13457066814104715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,16383,0.2425546646118164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,4095,0.07418666779994965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,16383,0.35816534360249835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,8191,0.19215466578801474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,32767,0.6876160303751627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,32767,0.45905065536499023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,float16,65535,1.3766560554504395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,16,128,1,float16,fp8,65535,0.9038080374399821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,3,0.18449600537618002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,1,0.17707200845082602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,3,0.23854400714238486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,1,0.2244853377342224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,7,0.2582133412361145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,7,0.19979733228683472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,15,0.25489066044489544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,31,0.30820266405741376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,15,0.2970400055249532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,63,0.2572159965833028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,31,0.255237340927124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,127,0.3914453188578288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,127,0.3081706762313843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,63,0.3128426671028137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,float16,255,0.6058773199717203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,16,128,1,float16,fp8,255,0.4708799918492635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,1,0.49198933442433673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,1,0.3771573305130005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,3,0.5046826601028442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,3,0.39069334665934247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,7,0.41149866580963135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,15,0.5040106773376465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,15,0.5872426827748617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,7,0.5110559860865275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,31,0.5039413372675577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,63,0.6160053412119547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,63,0.5060426791508993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,1,0.01423466702302297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,31,0.6079200108846029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,float16,127,0.771338701248169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,1,0.009082666908701261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,3,0.014368000129858652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,3,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,7,0.014218666901191076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,7,0.010309333602587381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,15,0.014757333944241205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,15,0.009770666559537252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,16,128,1,float16,fp8,127,0.6087040106455485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,31,0.01595199977358182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,31,0.011898666620254517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,63,0.01634666696190834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,127,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,127,0.01198400060335795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,255,0.01865600049495697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,63,0.01312000056107839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,255,0.013663999736309052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,511,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,511,0.02773333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,1023,0.05073600014050802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,1023,0.029461334149042766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,2047,0.056757330894470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,4095,0.10457600156466167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,4095,0.2055573264757792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,2047,0.11007466912269592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,8191,0.3959039847056071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,8191,0.19699732462565103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,16383,0.7749866644541422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,16383,0.3819520076115926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,1,0.01504533365368843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,fp8,32767,0.9290773073832194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,1,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,16,128,1,float16,float16,32767,1.5440266927083333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,3,0.019930666933457058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,3,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,7,0.019685332973798115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,7,0.015087999403476715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,31,0.022405333817005157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,15,0.015082667271296183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,15,0.020330666253964107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,31,0.01709866647919019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,63,0.02257599929968516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,63,0.017093333105246227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,127,0.022970666488011677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,127,0.01720000058412552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,255,0.026575999955336254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,511,0.04667733112970988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,511,0.02701333413521449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,1023,0.07340266803900401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,255,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,1023,0.04711999992529551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,2047,0.1623146633307139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,2047,0.08337066570917766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,4095,0.15808000167210898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,8191,0.5879466533660889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,4095,0.3062719901402791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,float16,16383,1.1574772993723552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,8191,0.3006773392359416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,12,12,128,1,float16,fp8,16383,0.5882399876912435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,1,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,1,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,3,0.008293333152929941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,3,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,7,0.008341333518425623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,7,0.010330666477481524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,15,0.009061333412925402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,31,0.009114666531483332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,31,0.010298666854699453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,63,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,255,0.01062400018175443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,127,0.010389333590865135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,127,0.012432000289360682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,255,0.012128000458081564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,511,0.01639466608564059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,511,0.016496000190575916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,1023,0.017738666385412216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,1023,0.017429333180189133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,2047,0.017642666896184284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,4095,0.017893332988023758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,4095,0.017621333400408428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,2047,0.01766933376590411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,16383,0.04029866556326548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,8191,0.02146666745344798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,8191,0.02204799900452296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,32767,0.07807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,16383,0.02868266652027766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,32767,0.0552106648683548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,1,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,1,0.008432000254591307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,fp8,65535,0.09635733564694722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,12,12,128,1,float16,float16,65535,0.1418186624844869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,3,0.008309333274761835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,3,0.01128000020980835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,15,0.009754666437705358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,7,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,15,0.008736000085870424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,63,0.009317333499590555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,31,0.009183999771873156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,31,0.010224000240365664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,127,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,127,0.012144000579913458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,255,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,1023,0.01791999985774358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,255,0.012175999581813812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,511,0.016586666305859882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,511,0.016778666526079178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,1023,0.01735466718673706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,4095,0.03746666759252548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,2047,0.018031999468803406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,2047,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,4095,0.036858665446440377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,8191,0.06351466476917267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,8191,0.05113600194454193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,16383,0.09053867061932881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,16383,0.1104800005753835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,32767,0.19674134254455566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,32767,0.156960000594457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,float16,65535,0.37098666032155353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,1,0.014314666390419006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,1,0.014101333916187286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,12,12,128,1,float16,fp8,65535,0.28940266370773315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,3,0.014293332894643148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,3,0.014202666779359182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,7,0.014858666807413101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,15,0.0162773331006368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,7,0.014645333091417948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,15,0.0170666662355264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,31,0.01695999999841054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,127,0.020090666910012562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,63,0.01728533332546552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,31,0.016261332978804905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,63,0.016255999604860943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,127,0.018496000518401463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,255,0.03625600039958954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,255,0.02647999922434489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,511,0.0737120012442271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,1023,0.14180266857147217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,511,0.04560533165931702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,1023,0.081386665503184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,2047,0.2702666719754537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,2047,0.15588800112406412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,4095,0.5252053340276083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,4095,0.2998186747233073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,float16,8191,1.0395840009053547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,1,0.008538666491707167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,12,12,128,1,float16,fp8,8191,0.5881760120391846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,1,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,3,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,3,0.008527999743819237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,7,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,7,0.008645333349704742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,15,0.008954666554927826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,31,0.009413333609700203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,15,0.009877333417534828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,63,0.009370666618148485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,63,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,127,0.009850666547815004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,127,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,255,0.0106133334338665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,255,0.012432000289360682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,511,0.01670933390657107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,1023,0.029535998900731403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,1023,0.029301332930723827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,511,0.01674666628241539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,2047,0.03772266705830892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,2047,0.03694933404525121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,4095,0.0510506679614385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,4095,0.062447999914487205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,8191,0.08927466471989949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,8191,0.11036800344785054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,16383,0.19434666633605957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,16383,0.1571999986966451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,32767,0.36897067228953045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,32767,0.2875093420346578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,fp8,65535,0.5519946813583374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,12,12,128,1,float16,float16,65535,0.7153279781341553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,1,0.024735999604066212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,1,0.02090666691462199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,3,0.025199999411900837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,3,0.021327999730904896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,7,0.02587199956178665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,7,0.0220320001244545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,15,0.029951999584833782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,15,0.027471999327341717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,31,0.030373332401116688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,31,0.02759466568628947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,63,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,63,0.03089066594839096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,127,0.04507733384768168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,127,0.032485333581765495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,255,0.06741333504517873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,255,0.051632001996040344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,1023,0.1513599952061971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,511,0.08396266897519429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,511,0.13596799969673157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,1023,0.27384533484776813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,2047,0.5292426745096842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,2047,0.29129600524902344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,1,0.045893331368764244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,float16,4095,1.037552038828532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,12,12,128,1,float16,fp8,4095,0.5658986568450928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,1,0.03772799919048945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,3,0.04666133224964142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,7,0.040192000567913055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,3,0.03894400099913279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,7,0.04750399788220724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,15,0.050666665037473045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,15,0.056261335810025535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,31,0.05723733206590017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,31,0.050570666790008545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,63,0.06505066653092702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,127,0.06477333108584087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,63,0.05087466537952423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,127,0.08205333352088928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,255,0.12581866979599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,255,0.09648000200589497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,511,0.2581920027732849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,511,0.15937599539756775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,float16,1023,0.5359306732813517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,12,12,128,1,float16,fp8,1023,0.29052799940109253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,1,0.08762133121490479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,1,0.0697920024394989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,3,0.08935466408729553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,3,0.07208533088366191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,7,0.09051199754079182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,7,0.07543999950091045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,15,0.10883200168609619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,15,0.09474133451779683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,31,0.11981333295504253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,31,0.09649599591890971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,63,0.12296533584594727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,127,0.1549066702524821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,63,0.1011253297328949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,127,0.12039466698964436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,255,0.24007999897003174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,1,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,255,0.1823093295097351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,1,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,float16,511,0.4914880196253459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,12,12,128,1,float16,fp8,511,0.3049866755803426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,3,0.008602666358153025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,7,0.008757333581646284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,3,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,15,0.00884799969693025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,7,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,15,0.0100426667680343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,31,0.010458666831254959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,127,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,63,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,31,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,63,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,127,0.012597333639860153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,255,0.011018666128317514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,511,0.02811199923356374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,511,0.027914665639400482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,255,0.0124746672809124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,1023,0.03257599969704946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,1023,0.03234666585922241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,2047,0.04888000090916952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,4095,0.08918933073679607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,2047,0.0435146689414978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,4095,0.07069866855939229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,8191,0.1585919956366221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,8191,0.12526933352152506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,16383,0.2922933300336202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,16383,0.22899200518925986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,32767,0.5599199930826823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,32767,0.43412800629933673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,float16,65535,1.1023626327514648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,12,12,128,1,float16,fp8,65535,0.8448586463928223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,1,0.16983999808629355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,1,0.13406933347384134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,3,0.17490132649739584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,3,0.13901866475741068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,7,0.1893440087636312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,7,0.14680533607800803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,15,0.2242506742477417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,31,0.2331413427988688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,15,0.19085333744684854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,63,0.23690666755040488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,63,0.1950933337211609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,31,0.1933493415514628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,127,0.2964586615562439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,127,0.2331999937693278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,float16,255,0.4601386785507202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,12,12,128,1,float16,fp8,255,0.3556373516718547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,1,0.35489598910013836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,3,0.28806400299072266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,1,0.2677599986394246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,3,0.3784693479537964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,7,0.3842080036799113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,7,0.3100000023841858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,15,0.37933866182963055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,15,0.44172799587249756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,31,0.45850133895874023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,31,0.37998934586842853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,63,0.4640586773554484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,1,0.01393066719174385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,1,0.009103999783595404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,63,0.3811093171437581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,float16,127,0.5820159912109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,3,0.014064000298579534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,3,0.009328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,12,12,128,1,float16,fp8,127,0.4587786595026652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,7,0.009178666397929192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,7,0.0138026662170887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,15,0.01440000037352244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,15,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,31,0.015664000064134598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,63,0.01588800052801768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,63,0.01321600005030632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,127,0.016176000237464905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,127,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,1023,0.04779199759165446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,255,0.018197332819302876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,255,0.013631999492645264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,511,0.02500266581773758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,511,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,1023,0.02940266579389572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,2047,0.09498133261998494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,2047,0.05406400064627329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,4095,0.1851573387781779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,8191,0.1880906621615092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,4095,0.09898666540781657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,8191,0.35204799969991046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,16383,0.36291734377543133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,16383,0.6893653074900309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,1,0.014149333039919535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,float16,32767,1.372938632965088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,1,0.009232000137368837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,12,12,128,1,float16,fp8,32767,0.8851040204366049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,3,0.009301333377758661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,3,0.014394666999578476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,7,0.01414399966597557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,7,0.009322666873534521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,15,0.014730667074521383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,15,0.009872000043590864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,31,0.015930666277805965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,31,0.011765333513418833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,127,0.016496000190575916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,63,0.012144000579913458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,127,0.01211200033624967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,255,0.018858666221300762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,255,0.013568000247081121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,63,0.0163680004576842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,511,0.026975999275843304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,511,0.018901333212852478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,1023,0.029535998900731403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,1023,0.05172266562779745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,2047,0.056757330894470215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,2047,0.11134399970372517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,4095,0.10424000024795532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,8191,0.39354666074117023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,4095,0.20548800627390543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,8191,0.19629865884780884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,1,0.008453333129485449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,16383,0.7716960112253824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,16383,0.38175467650095624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,3,0.008336000144481659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,float16,32767,1.5458347002665203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,1,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,8,128,1,float16,fp8,32767,0.9186986287434896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,3,0.009322666873534521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,7,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,15,0.008527999743819237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,15,0.009872000043590864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,63,0.009359999870260557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,31,0.00903466654320558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,31,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,127,0.012202666451533636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,63,0.012015999605258306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,127,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,255,0.01211200033624967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,255,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,511,0.016656000167131424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,511,0.016373333831628162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,2047,0.017450666675964992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,1023,0.017653333644072216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,1023,0.0174346665541331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,2047,0.01747200017174085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,4095,0.017797333498795826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,8191,0.029445332785447437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,4095,0.017594666530688603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,16383,0.04099733382463455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,8191,0.029658667743206024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,16383,0.040192000567913055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,32767,0.0816480020682017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,32767,0.06233599781990051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,65535,0.14548800388971964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,65535,0.11740266283353169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,1,0.008256000156203905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,1,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,float16,131071,0.271781325340271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,8,128,1,float16,fp8,131071,0.214464008808136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,3,0.008527999743819237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,3,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,7,0.00938666673998038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,15,0.009850666547815004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,7,0.008330666770537695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,15,0.008432000254591307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,63,0.009328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,31,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,31,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,63,0.0122079998254776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,255,0.012202666451533636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,127,0.009413333609700203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,255,0.010437333335479101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,127,0.012047999848922094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,511,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,511,0.016517333686351776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,1023,0.01740266631046931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,1023,0.017605333278576534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,2047,0.01812800019979477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,2047,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,4095,0.029893333713213604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,4095,0.02957333376010259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,8191,0.04119466741879781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,8191,0.04026666780312856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,16383,0.06137600044409434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,16383,0.08049599826335907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,32767,0.1411679983139038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,65535,0.2574133276939392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,32767,0.11378666758537292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,65535,0.20124266544977823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,float16,131071,0.4905173381169637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,1,0.013754667093356451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,1,0.01533866673707962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,8,128,1,float16,fp8,131071,0.37825600306193036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,3,0.014127999544143677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,3,0.015301333119471868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,7,0.013893333574136099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,7,0.015423999478419622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,15,0.014389333625634512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,31,0.014752000570297241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,15,0.016597333053747814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,31,0.020261333634455998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,63,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,63,0.02027733375628789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,127,0.017338667064905167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,255,0.02607999990383784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,127,0.02033599962790807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,255,0.02348800003528595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,511,0.06178133189678192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,511,0.035530666510264076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,1023,0.11411199967066447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,1023,0.05923733115196228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,2047,0.21921066443125406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,2047,0.10770666599273682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,4095,0.4195520083109538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,4095,0.20104533433914185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,8191,0.8521493275960287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,8191,0.3845813274383545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,fp8,16383,0.7554506460825602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,8,128,1,float16,float16,16383,1.530666669209798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,1,0.008458666503429413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,1,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,7,0.00984533317387104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,3,0.008746666833758354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,3,0.009818666925032934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,7,0.008661333471536636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,15,0.008687999720374743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,15,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,31,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,31,0.010501333822806677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,63,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,63,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,127,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,127,0.012346666306257248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,511,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,255,0.010741333166758219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,255,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,2047,0.029637334247430164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,511,0.016805333395799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,2047,0.029706666866938274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,1023,0.017957333475351334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,1023,0.017488000293572743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,4095,0.0409706657131513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,4095,0.039781334499518074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,8191,0.07813866436481476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,8191,0.05972266693909963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,16383,0.1290880044301351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,16383,0.10419733325640361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,32767,0.23220266898473105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,32767,0.18241065740585327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,65535,0.4425813357035319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,65535,0.3463573455810547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,1,0.015168000012636185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,1,0.019215999792019527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,3,0.019530666371186573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,float16,131071,0.867690642674764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,7,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,8,128,1,float16,fp8,131071,0.6742239793141683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,3,0.015504000087579092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,7,0.015962666521469753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,31,0.019845332950353622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,15,0.01971199984351794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,15,0.021589333812395733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,63,0.02213866760333379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,31,0.02216000109910965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,63,0.01972266659140587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,127,0.023386667172114056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,511,0.09964799880981445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,127,0.028165332973003387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,255,0.04905066887537638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,255,0.03490666548411051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,511,0.05881600081920624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,1023,0.19422932465871176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,2047,0.37145598729451496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,2047,0.20016533136367798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,4095,0.7229653199513754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,4095,0.38678932189941406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,1023,0.1053600013256073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,1,0.03422933320204417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,float16,8191,1.4286079406738281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,8,128,1,float16,fp8,8191,0.7689812978108724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,1,0.026362667481104534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,7,0.02796799937884013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,3,0.03500800083080927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,3,0.02737066646416982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,15,0.039290666580200195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,31,0.035375999907652535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,7,0.03533333291610082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,31,0.03978666663169861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,15,0.03537066777547201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,63,0.042965332667032875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,63,0.035573333501815796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,127,0.0432533323764801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,127,0.059989333152770996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,255,0.06645866731802623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,511,0.18867733081181845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,1023,0.19929067293802896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,511,0.10922132929166158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,255,0.09134933352470398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,1023,0.37652266025543213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,float16,2047,0.7356159687042236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,1,0.04806933303674062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,8,128,1,float16,fp8,2047,0.3833119869232178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,3,0.062047998110453285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,1,0.060922667384147644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,7,0.06272533535957336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,15,0.07256000240643819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,7,0.052245333790779114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,15,0.06541866560777028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,3,0.049914668003718056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,31,0.07891733447710673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,63,0.06834666430950165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,63,0.08505066235860188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,31,0.06520000100135803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,127,0.10750400026639302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,127,0.08326933284600575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,511,0.3460640112559001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,255,0.12501866618792215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,511,0.2079520026842753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,1,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,255,0.16643200318018594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,fp8,1023,0.3850613435109456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,1,0.00842666688064734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,8,128,1,float16,float16,1023,0.7133653163909912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,7,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,3,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,3,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,31,0.010618666807810465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,15,0.009888000165422758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,15,0.009061333412925402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,31,0.01051733394463857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,7,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,63,0.009701333319147428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,127,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,63,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,127,0.01227733368674914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,255,0.010869332899649939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,1023,0.02993600070476532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,511,0.016938666502634685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,255,0.01219733307758967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,1023,0.029669334491093952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,4095,0.06679466863473256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,2047,0.03827733298142751
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,511,0.017344000438849132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,2047,0.037061333656311035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,16383,0.2059733271598816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,4095,0.05236800014972687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,8191,0.09066133697827657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,8191,0.11587199568748474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,16383,0.16040000319480896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,32767,0.2978399991989136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,32767,0.3843573331832886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,65535,0.741925319035848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,65535,0.5645920038223267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,1,0.11458667119344075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,float16,131071,1.3717494010925293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,1,0.09142933289210002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,3,0.09442133704821269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,8,128,1,float16,fp8,131071,0.907093365987142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,3,0.11674666404724121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,7,0.11909332871437073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,7,0.09897599617640178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,15,0.1255466639995575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,15,0.15001599987347922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,31,0.15762666861216226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,63,0.1609119971593221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,31,0.13030933340390524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,63,0.13264532883961996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,127,0.2014453411102295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,127,0.15888532996177673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,255,0.31250133117039997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,255,0.24032533168792725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,1,0.22409600019454956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,float16,511,0.6553226709365845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,1,0.17707733313242593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,8,128,1,float16,fp8,511,0.4039359887440999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,3,0.18452266852060953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,3,0.24034667015075684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,7,0.1993066668510437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,7,0.25862934192021686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,15,0.2548373341560364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,15,0.2971786657969157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,31,0.30823999643325806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,31,0.25540266434351605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,63,0.3129653334617615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,63,0.25697600841522217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,1,0.008634666601816813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,1,0.009989333028594652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,127,0.308463990688324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,127,0.39161598682403564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,3,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,7,0.008725333337982496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,7,0.009882666791478792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,3,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,fp8,255,0.47098131974538165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,15,0.010165333126982054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,8,128,1,float16,float16,255,0.6062560081481934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,15,0.010112000008424124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,31,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,63,0.009882666791478792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,127,0.009935999910036722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,63,0.012485332787036896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,127,0.012485332787036896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,255,0.011098666737476984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,255,0.012522666404644648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,1023,0.03379199902216593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,511,0.028815999627113342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,511,0.028234665592511494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,1023,0.03268266717592875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,2047,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,2047,0.055546666185061135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,4095,0.07381333410739899
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,8191,0.1918613314628601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,4095,0.1090133289496104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,8191,0.1341919998327891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,16383,0.35972265402475995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,16383,0.24265066782633463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,32767,0.45861868063608807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,32767,0.6935893694559733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,1,0.008879999940594038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,float16,65535,1.371605396270752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,8,128,1,float16,fp8,65535,0.8988479773203532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,3,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,3,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,7,0.009877333417534828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,15,0.009904000287254652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,7,0.009850666547815004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,15,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,31,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,127,0.00984533317387104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,63,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,31,0.010661333799362183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,255,0.011152000476916632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,63,0.012602667013804117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,127,0.012597333639860153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,511,0.02870933214823405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,255,0.012634667257467905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,511,0.028042666614055634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,1023,0.03378133227427801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,4095,0.1083626647790273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,1023,0.032645332316557564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,2047,0.04498666524887085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,2047,0.05537599821885427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,4095,0.07509866853555043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,8191,0.19193599621454874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,16383,0.35766398906707764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,8191,0.13456533352533975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,16383,0.2416586677233378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,1,0.008469333251317343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,32767,0.6874773502349854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,1,0.009306666751702627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,32767,0.45980266729990643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,fp8,65535,0.8977599938710531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,4,128,1,float16,float16,65535,1.3753706614176433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,3,0.009301333377758661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,3,0.008197333042820295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,7,0.008176000167926153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,31,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,7,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,15,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,31,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,15,0.008639999975760778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,63,0.009269333134094873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,127,0.009173333023985228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,63,0.012117333710193634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,127,0.012069333344697952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,255,0.010431999961535135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,511,0.016421332955360413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,255,0.01209066684047381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,1023,0.01729600007335345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,511,0.01613333324591319
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,1023,0.017344000438849132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,2047,0.01729600007335345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,2047,0.01754133279124896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,4095,0.017658667018016178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,8191,0.017770666629076004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,4095,0.017429333180189133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,8191,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,16383,0.017664000391960144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,16383,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,32767,0.02309866746266683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,32767,0.024442667762438457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,65535,0.056261335810025535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,65535,0.03421333432197571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,float16,131071,0.10126933455467224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,1,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,3,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,1,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,3,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,4,128,1,float16,fp8,131071,0.07032533486684163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,7,0.008346666892369589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,7,0.009375999992092451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,15,0.008623999853928884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,15,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,31,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,31,0.010181333248813948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,63,0.009359999870260557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,127,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,255,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,63,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,255,0.012074666718641916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,511,0.016480000068744022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,1023,0.017690667261679966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,511,0.016282666474580765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,2047,0.01752000053723653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,1023,0.01740266631046931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,2047,0.01756799966096878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,8191,0.029690665503342945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,4095,0.017898666361967724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,16383,0.0410453329483668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,8191,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,4095,0.017637333522240322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,16383,0.04018666595220566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,32767,0.08242133259773254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,32767,0.06230400005976359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,65535,0.11735999584197998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,65535,0.14299199978510538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,1,0.014053333550691605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,1,0.009178666397929192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,fp8,131071,0.214026669661204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,4,128,1,float16,float16,131071,0.27037866910298664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,3,0.014170666535695394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,3,0.00922133338948091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,7,0.01421333352724711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,7,0.00926399976015091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,15,0.014581333845853806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,15,0.009888000165422758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,31,0.015935999651749928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,31,0.011909333368142446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,63,0.016048000504573185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,63,0.012058666596810022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,127,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,511,0.027562665442625683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,127,0.011994666109482447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,255,0.018794666975736618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,255,0.01360000049074491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,511,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,1023,0.05146133402983347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,2047,0.11193066835403442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,1023,0.029525332152843475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,2047,0.05665599803129832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,4095,0.20560532808303833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,4095,0.1046506663163503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,8191,0.39346667130788165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,16383,0.7708053588867188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,8191,0.19607466459274292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,16383,0.38142398993174237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,1,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,fp8,32767,0.9144426981608073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,4,128,1,float16,float16,32767,1.545749346415202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,3,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,3,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,7,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,31,0.009402666861812273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,15,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,15,0.009770666559537252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,7,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,31,0.010485333700974783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,127,0.009530666594703993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,63,0.012186666329701742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,63,0.009237333511312803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,255,0.010373333469033241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,511,0.01684800038735072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,127,0.012128000458081564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,511,0.01657066618402799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,255,0.012159999459981918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,1023,0.017738666385412216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,1023,0.017658667018016178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,4095,0.029482667644818623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,2047,0.017898666361967724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,2047,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,4095,0.029882666965325672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,16383,0.06132799883683523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,8191,0.04121066629886627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,8191,0.04033066580692927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,16383,0.07977066437403361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,32767,0.14124799768129984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,32767,0.11453333497047424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,65535,0.20080532630284628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,65535,0.2569226622581482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,1,0.013642666240533194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,1,0.015306666493415833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,fp8,131071,0.37701865037282306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,3,0.014090667168299357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,4,128,1,float16,float16,131071,0.489957332611084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,3,0.015552000453074774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,7,0.013866666704416275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,7,0.015392000476519266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,15,0.014490666488806406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,15,0.01685333376129468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,31,0.01470400020480156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,63,0.01524266724785169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,31,0.02029866725206375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,511,0.06102400024731954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,63,0.020165332903464634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,127,0.017551999539136887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,127,0.020362666497627895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,255,0.02608533451954524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,255,0.023685333629449207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,511,0.035258665680885315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,1023,0.11395200093587239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,1023,0.05955199897289276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,2047,0.10641599694887798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,2047,0.22211732467015585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,4095,0.42262399196624756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,4095,0.20186134179433188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,8191,0.3847786585489909
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,1,0.019333332777023315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,8191,0.8491786321004232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,1,0.01498666654030482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,3,0.01970133309563001
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,3,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,fp8,16383,0.7559786637624105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,4,128,1,float16,float16,16383,1.538991928100586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,7,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,7,0.015962666521469753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,31,0.02199466774861018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,15,0.021727999051411945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,15,0.019909333437681198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,31,0.01989866668979327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,63,0.022197333474953968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,127,0.0233599990606308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,63,0.019834666202465694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,127,0.028117333849271137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,255,0.049253334601720176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,255,0.03533866753180822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,511,0.09962133566538493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,511,0.05890666445096334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,1023,0.19489065806070963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,1023,0.10498666763305664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,2047,0.37117866675059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,2047,0.19944000244140625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,1,0.02647999922434489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,1,0.034186666210492454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,float16,4095,0.7239253520965576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,4,128,1,float16,fp8,4095,0.3871893485387166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,3,0.0349440003434817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,3,0.027280000348885853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,7,0.02810666710138321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,7,0.03537066777547201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,15,0.03922666609287262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,15,0.03551466763019562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,31,0.039647998909155525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,31,0.03533866753180822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,63,0.03549866626660029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,127,0.0432586669921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,127,0.06016000111897787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,63,0.04381866753101349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,255,0.09147733449935913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,255,0.06669866542021434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,511,0.18793600797653198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,511,0.10955199599266052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,1023,0.1986560026804606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,1,0.008383999889095625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,1023,0.3781866629918416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,3,0.008586666857202848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,fp8,2047,0.38260265191396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,1,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,4,128,1,float16,float16,2047,0.7341600259145101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,15,0.008762666955590248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,7,0.008469333251317343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,3,0.009733333562811216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,15,0.009733333562811216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,7,0.010618666807810465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,31,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,31,0.01051733394463857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,63,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,127,0.009541333342591921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,63,0.010453333457310995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,511,0.016800000021855038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,127,0.012416000167528788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,255,0.01146666705608368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,255,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,2047,0.02991466720898946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,511,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,1023,0.01788266624013583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,2047,0.0296426663796107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,4095,0.04142399877309799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,1023,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,4095,0.04002666721741358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,8191,0.0773119976123174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,8191,0.05930666625499725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,16383,0.12822399536768594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,16383,0.10337066650390625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,32767,0.23309866587320963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,65535,0.44235201676686603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,65535,0.34545600414276123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,32767,0.18186666568120322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,1,0.06098666787147522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,3,0.05003199974695841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,float16,131071,0.8657066822052002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,4,128,1,float16,fp8,131071,0.6710560321807861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,3,0.06217066446940104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,1,0.048170665899912514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,7,0.0625600020090739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,15,0.07287999987602234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,7,0.05209066470464071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,15,0.06548266609509786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,31,0.0794239987929662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,31,0.06553600231806438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,63,0.08559466401735942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,127,0.10858133435249329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,63,0.06769066552321117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,127,0.08287466565767924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,255,0.1666933298110962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,255,0.12499733765920003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,511,0.3446720043818156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,511,0.20879467328389487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,float16,1023,0.7150239944458008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,1,0.11436800161997478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,1,0.09149332841237386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,4,128,1,float16,fp8,1023,0.38464534282684326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,3,0.11649066209793091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,3,0.09468266367912292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,7,0.09902933239936829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,15,0.15029333035151163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,7,0.11893866459528606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,15,0.12531200051307678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,31,0.1579253375530243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,31,0.13033599654833475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,63,0.13261333107948303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,63,0.16133866707483926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,127,0.20241600275039673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,127,0.1586133340994517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,255,0.3131093382835388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,255,0.2404373288154602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,1,0.009866666669646898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,float16,511,0.6519946654637655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,7,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,3,0.009466666728258133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,7,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,4,128,1,float16,fp8,511,0.4040693442026774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,15,0.009002666920423508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,31,0.009610666582981745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,63,0.012421333541472753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,31,0.010602666685978571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,63,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,127,0.009930666536092758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,127,0.012309333930412928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,255,0.010634666929642359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,255,0.012250666817029318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,511,0.017184000462293625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,511,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,1023,0.02962133288383484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,2047,0.03708266715208689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,1023,0.029802667597929638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,2047,0.038831998904546104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,4095,0.06577066580454509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,4095,0.0525546669960022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,16383,0.20629332462946573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,8191,0.09096533060073853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,8191,0.11457600196202596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,16383,0.16060266892115274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,32767,0.4036053419113159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,32767,0.2956906755765279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,65535,0.741002639134725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,65535,0.563098669052124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,float16,131071,1.3719946543375652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,4,128,1,float16,fp8,131071,0.9078293641408285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,1,0.04640533526738485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,1,0.05968533456325531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,3,0.06072533130645752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,7,0.0613919993241628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,3,0.047775998711586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,7,0.05006400247414907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,15,0.0702453354994456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,63,0.07133333384990692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,31,0.07081066568692525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,31,0.06318933268388112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,15,0.06322133541107178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,63,0.06330666442712148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,255,0.126720001300176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,127,0.08519466718037923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,127,0.07579199969768524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,255,0.11561066905657451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,511,0.21066667636235556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,1023,0.3805493513743083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,511,0.1967466672261556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,1023,0.3593920071919759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,2047,0.7164053122202555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,2047,0.6837173302968343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,float16,4095,1.4092586835225422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,1,128,1,float16,fp8,4095,1.3257973194122314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,3,0.06073066592216492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,1,0.059445331494013466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,1,0.04650133351484934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,3,0.047925333182017006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,15,0.06366933385531108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,7,0.061381335059801735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,7,0.05055999755859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,15,0.07109866539637248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,31,0.07172800103823344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,31,0.06367466847101848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,63,0.07234666744867961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,63,0.06385600070158641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,127,0.08590400218963623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,127,0.07576000193754832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,255,0.12869866689046225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,255,0.11585600177447002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,1023,0.3826506535212199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,511,0.2125599980354309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,511,0.19733333587646484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,1023,0.36006398995717365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,2047,0.7343466281890869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,2047,0.6837973594665527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,fp8,4095,1.3287359873453777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,2,128,1,float16,float16,4095,1.4234879811604817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,1,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,3,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,7,0.009375999992092451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,15,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,31,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,63,0.009258666386206945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,1,0.01331199953953425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,3,0.01328533391157786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,127,0.0129120002190272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,7,0.01309866706530253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,15,0.013125333935022354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,255,0.02128533273935318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,255,0.026474667092164356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,511,0.03453866640726725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,31,0.012949333836634954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,1023,0.05807999769846598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,63,0.013013333082199097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,127,0.010175999874869982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,2047,0.08295466502507527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,float16,4095,0.10859733819961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,2047,0.11150933305422465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,4095,0.13870933651924133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,511,0.039893334110577904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,4,128,1,float16,fp8,1023,0.0766186664501826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,1,0.007765333478649457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,3,0.007706666365265846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,7,0.008378666515151659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,15,0.007765333478649457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,31,0.00785600021481514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,1,0.01643199970324834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,63,0.007936000203092894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,31,0.016506666938463848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,3,0.016490666816631954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,63,0.016554666062196095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,7,0.01658133293191592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,127,0.016415999581416447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,127,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,255,0.03136000037193298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,15,0.016554666062196095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,511,0.04896533489227295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,1023,0.06366399923960368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,1023,0.04560000201066335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,2047,0.08699199557304382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,255,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,1,0.008469333251317343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,float16,4095,0.12865599989891052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,3,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,1,0.009610666582981745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,511,0.03211733450492223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,7,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,2047,0.06002133091290792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,3,0.009530666594703993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,7,0.008725333337982496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,15,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,15,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,128,8,128,1,float16,fp8,4095,0.08777599533398946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,31,0.009519999846816063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,31,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,63,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,255,0.01227733368674914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,63,0.012261333564917246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,127,0.012250666817029318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,127,0.009648000200589498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,255,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,1023,0.032618666688601174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,511,0.028245332340399425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,1023,0.03253333270549774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,511,0.027888000011444092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,2047,0.0441599984963735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,2047,0.0436160018046697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,4095,0.0662720004717509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,4095,0.06520533561706543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,8191,0.11019200086593628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,8191,0.10776533683141072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,3,0.008474666625261307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,float16,16383,0.19788267215092978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,1,128,1,float16,fp8,16383,0.19260267416636148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,1,0.00850133349498113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,1,0.00973866693675518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,7,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,3,0.00973866693675518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,7,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,15,0.008687999720374743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,31,0.009514666472872099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,15,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,31,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,63,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,63,0.012319999436537424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,511,0.028069332242012024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,127,0.009583999713261923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,255,0.0107893335322539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,511,0.028250666956106823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,127,0.012272000312805176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,255,0.012319999436537424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,1023,0.03260799994071325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,2047,0.044079999128977455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,1023,0.032842665910720825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,2047,0.04385066529115041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,4095,0.06614933411280315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,8191,0.11013333002726237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,4095,0.06505600114663442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,8191,0.10758399963378906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,7,0.00855466661353906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,fp8,16383,0.19259732961654663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,2,128,1,float16,float16,16383,0.19818667570749918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,1,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,1,0.008565333361426989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,3,0.008703999842206636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,3,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,7,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,15,0.008789333204428354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,31,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,31,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,63,0.00980266680320104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,63,0.012234666695197424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,127,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,1023,0.03272533416748047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,255,0.0106133334338665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,127,0.012485332787036896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,511,0.028309332827727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,255,0.012389333297808966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,15,0.009829333052039146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,511,0.027999999622503918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,1023,0.032613334556420646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,2047,0.016010666886965435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,2047,0.017423999806245167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,4095,0.018309333672126133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,4095,0.018618666877349217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,8191,0.023354666928450268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,float16,16383,0.02548266698916753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,8191,0.022181332111358643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,7,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,4,128,1,float16,fp8,16383,0.027029333015282948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,1,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,1,0.00850133349498113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,3,0.00871999996403853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,3,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,7,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,15,0.008661333471536636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,15,0.009919999788204828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,31,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,31,0.010351999973257383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,127,0.012426666915416718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,63,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,63,0.012341332932313284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,511,0.02802666773398717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,255,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,255,0.0124746672809124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,1023,0.01402666668097178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,511,0.028165332973003387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,1023,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,127,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,2047,0.015477333217859268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,2047,0.01179733375708262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,4095,0.01351999988158544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,4095,0.016314666718244553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,16383,0.02252800017595291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,8191,0.02604266752799352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,float16,16383,0.031040000418821972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,128,8,128,1,float16,fp8,8191,0.018650667121013004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,1,0.013888000200192133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,1,0.008954666554927826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,3,0.008896000062425932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,3,0.013999999811251959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,7,0.014053333550691605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,7,0.009077333534757296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,15,0.014271999398867289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,31,0.011626667032639185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,15,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,31,0.01569066693385442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,63,0.01591466615597407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,255,0.018085333208243053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,63,0.011722666521867117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,127,0.015829333414634068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,127,0.011866666376590729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,255,0.013477332890033722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,1023,0.036730666955312095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,511,0.024453334510326385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,511,0.018735999862353008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,1023,0.02920000006755193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,2047,0.050144001841545105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,2047,0.06127466758092245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,4095,0.09152533610661824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,4095,0.11090133587519328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,8191,0.20896534125010172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,8191,0.17434666554133096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,1,0.009050666665037474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,float16,16383,0.4071199893951416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,1,0.013701333353916803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,1,128,1,float16,fp8,16383,0.3389333486557007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,3,0.014096000542243322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,7,0.009109333157539368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,3,0.009072000160813332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,7,0.01393066719174385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,63,0.01589866727590561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,15,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,31,0.015530666957298914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,15,0.014282666146755219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,31,0.011887999872366587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,63,0.011850666254758835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,127,0.016143999993801117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,127,0.011711999773979187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,255,0.01357866699496905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,255,0.018122666825850803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,511,0.024314666787783306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,511,0.018922666708628338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,2047,0.06150400141874949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,1023,0.036917333801587425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,1023,0.02939733366171519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,8191,0.1740106741587321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,2047,0.050069332122802734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,4095,0.09144533673922221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,4095,0.11053867141405742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,8191,0.21073599656422934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,float16,16383,0.4105386734008789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,2,128,1,float16,fp8,16383,0.33921066919962567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,1,0.01370666672786077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,1,0.008885333314538002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,15,0.009839999799927076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,3,0.01393066719174385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,3,0.009130666653315226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,7,0.014042666802803675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,15,0.014314666390419006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,7,0.009098666409651438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,31,0.011717333147923151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,31,0.015605332950750986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,63,0.011658667276302973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,63,0.015941333025693893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,127,0.016154666741689045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,127,0.011973333855470022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,255,0.01821333294113477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,255,0.013584000368913015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,511,0.024314666787783306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,4095,0.020143999407688778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,511,0.018837332725524902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,2047,0.018687999496857326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,1023,0.016149333367745083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,2047,0.015967999895413715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,1023,0.013882666826248169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,8191,0.031514666974544525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,4095,0.018464000274737675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,16383,0.04022400081157684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,float16,16383,0.03958400090535482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,4,128,1,float16,fp8,8191,0.032586666444937386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,3,0.01381333296497663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,1,0.013946666071812311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,1,0.008890666688481966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,3,0.009189333145817121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,15,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,7,0.009141333401203156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,7,0.013797332843144735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,15,0.01423466702302297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,31,0.015696000307798386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,31,0.011920000116030375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,63,0.011781333635250727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,63,0.01584533353646596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,511,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,255,0.01809599995613098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,127,0.011882666498422623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,127,0.016117333124081295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,511,0.01332266628742218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,255,0.01357866699496905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,1023,0.013807999591032663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,1023,0.010570666442314783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,4095,0.015840000162522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,2047,0.02107733239730199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,2047,0.013199999928474426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,8191,0.03658133248488108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,4095,0.024656000236670177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,8191,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,float16,16383,0.05448000133037567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,128,8,128,1,float16,fp8,16383,0.03588266670703888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,1,0.11218667030334473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,7,0.09497599800427754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,3,0.11463466286659241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,3,0.09102400143941243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,7,0.11636799573898315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,1,0.08797333637873332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,31,0.13686399658521017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,15,0.13617600003878275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,63,0.12128532926241557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,63,0.1381066640218099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,31,0.12117333213488261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,127,0.1641973356405894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,127,0.1449066698551178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,15,0.12115733822186787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,255,0.24659200509389242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,255,0.22292266289393106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,511,0.41122666994730633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,1023,0.7428906758626302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,511,0.3816959857940674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,1023,0.701311985651652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,1,0.11202133695284526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,1,0.08832533160845439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,7,0.11597866813341777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,float16,2047,1.416810671488444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,3,0.1147093375523885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,1,128,1,float16,fp8,2047,1.336586634318034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,3,0.09101866682370503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,7,0.09570133686065674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,15,0.12168533603350322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,15,0.13689600427945456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,31,0.12167466680208842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,63,0.13961600263913473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,31,0.13845333456993103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,63,0.12210133671760559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,127,0.16607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,127,0.1448319951693217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,255,0.24942400058110556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,255,0.22298665841420492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,1023,0.7626026471455892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,511,0.41307199001312256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,511,0.38235199451446533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,1023,0.7020479838053385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,1,0.01628799984852473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,fp8,2047,1.338762601216634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,1,0.0201706662774086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,2,128,1,float16,float16,2047,1.431839942932129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,3,0.020917333662509918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,7,0.016250666230916977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,3,0.01600533351302147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,7,0.020394666741291683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,15,0.016042667130629223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,15,0.020517333100239437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,31,0.01624533285697301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,31,0.020256000260512035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,63,0.020421333611011505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,127,0.020341333001852036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,63,0.016106666376193363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,127,0.01770666614174843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,511,0.06070399781068166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,255,0.03914133210976919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,255,0.038746667404969536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,1023,0.07890666524569194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,2047,0.14004799723625183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,float16,2047,0.10998400052388509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,1023,0.11214933792750041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,1,0.027722666660944622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,4,128,1,float16,fp8,511,0.07693333427111308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,3,0.012991999586423239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,1,0.013023999830087027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,3,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,7,0.026965332527955372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,31,0.027061333258946735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,15,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,31,0.013082666943470636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,15,0.014015999933083853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,63,0.026890667776266735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,7,0.012928000340859095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,63,0.012928000340859095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,127,0.026933332284291584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,255,0.05295466880003611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,127,0.01523200049996376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,255,0.03286933402220408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,511,0.06705600023269653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,511,0.04629333317279816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,1023,0.08641067147254944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,1023,0.06111466884613037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,float16,2047,0.1328480045000712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,1,0.01339200014869372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,1,0.014709333578745524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,3,0.01340266689658165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,128,8,128,1,float16,fp8,2047,0.08877333005269368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,3,0.014922666052977243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,7,0.013568000247081121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,15,0.014197333405415217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,7,0.015103999525308609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,15,0.016229332735141117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,31,0.014277332772811254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,63,0.014314666390419006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,31,0.019946667055288952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,127,0.0174346665541331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,63,0.019882666567961376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,127,0.020101333657900494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,255,0.02239466706911723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,255,0.023168000082174938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,511,0.03454400102297465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,511,0.03387733300526937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,1023,0.059402664502461754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,1023,0.054058666030565895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,2047,0.10812266667683919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,2047,0.09448533256848653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,4095,0.17549866437911987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,4095,0.20574400822321573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,8191,0.33818666140238446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,8191,0.4023840030034383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,1,0.013536000003417334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,fp8,16383,0.6629813512166342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,1,128,1,float16,float16,16383,0.8046720027923584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,1,0.014815999815861383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,3,0.01357866699496905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,3,0.015077333897352219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,7,0.014949332922697067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,15,0.01626666635274887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,15,0.014271999398867289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,31,0.014309333016475042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,7,0.013925333817799887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,31,0.01989866668979327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,63,0.0144213338692983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,127,0.016522667060295742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,63,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,127,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,255,0.022330666581789654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,511,0.03461866577466329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,255,0.023311999936898548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,511,0.03387200087308884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,1023,0.059061333537101746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,1023,0.05402133365472158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,2047,0.10798933108647664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,4095,0.21034133434295654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,2047,0.09450667103131612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,4095,0.1755733291308085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,8191,0.3378133376439412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,8191,0.4081546862920125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,float16,16383,0.977013349533081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,2,128,1,float16,fp8,16383,0.662773331006368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,1,0.01349866638580958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,3,0.013776000589132309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,3,0.01488000030318896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,7,0.013616000612576803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,7,0.015178666760524115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,15,0.014090667168299357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,15,0.016224000602960587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,31,0.014277332772811254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,31,0.0199946661790212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,63,0.014346666634082794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,63,0.020047999918460846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,127,0.016650666793187458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,127,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,255,0.022517333428064983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,511,0.012693333129088083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,255,0.02334933231274287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,1023,0.016016000260909397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,511,0.015541333705186844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,1023,0.014005333185195923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,2047,0.02573866645495097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,8191,0.047930667797724404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,2047,0.024533333877722423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,4095,0.030752000709374745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,float16,16383,0.06764799853165944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,4095,0.028207999964555103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,16383,0.07700799902280171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,4,128,1,float16,fp8,8191,0.04457599918047587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,1,0.013541333377361298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,3,0.013674666484196981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,3,0.015109332899252573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,1,0.014853333433469137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,7,0.01523200049996376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,15,0.01651200031240781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,7,0.01357866699496905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,15,0.014117332796255747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,127,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,31,0.019946667055288952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,31,0.014266667266686758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,63,0.014271999398867289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,63,0.020101333657900494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,127,0.020015999674797058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,255,0.01328533391157786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,255,0.009189333145817121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,511,0.013594667116800943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,511,0.01198400060335795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,4095,0.04262933135032654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,1023,0.011973333855470022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,2047,0.029093332588672638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,2047,0.01989866668979327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,1023,0.019760000209013622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,4095,0.023845332364241283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,8191,0.04072533299525579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,8191,0.05975466469923655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,float16,16383,0.09132267038027446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,128,8,128,1,float16,fp8,16383,0.06341333190600078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,3,0.176362673441569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,1,0.21979733308156332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,3,0.224400003751119
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,1,0.17084266742070517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,7,0.22694933414459229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,15,0.23704532782236734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,15,0.266975998878479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,7,0.18410134315490723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,31,0.26873600482940674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,31,0.2368533412615458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,63,0.27027734120686847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,127,0.3228213389714559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,63,0.2375040054321289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,127,0.2840426762898763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,255,0.4853813250859578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,255,0.4377280076344808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,511,0.8154079914093018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,511,0.7521386941274008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,float16,1023,1.4865226745605469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,1,128,1,float16,fp8,1023,1.385503927866618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,1,0.21947733561197916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,1,0.1716746687889099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,3,0.22470933198928833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,3,0.17627733945846558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,7,0.22698666652043661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,15,0.23802665869394937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,15,0.2685226599375407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,7,0.18581867218017578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,31,0.27163734038670856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,31,0.23771733045578003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,63,0.2736266652743022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,63,0.23869333664576212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,127,0.28384532531102497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,127,0.32627199093500775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,255,0.4378986756006877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,255,0.49484801292419434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,1023,1.506831963857015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,float16,511,0.8339520295461019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,1,0.029391999046007793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,1023,1.3890719413757324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,1,0.0349386657277743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,2,128,1,float16,fp8,511,0.7538666725158691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,3,0.03494933247566223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,3,0.029274667302767437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,7,0.03531199942032496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,15,0.034847999612490334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,31,0.02942399928967158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,15,0.029232000311215717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,7,0.029391999046007793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,31,0.03479466587305069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,63,0.03472000112136205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,127,0.03266133368015289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,127,0.035274667044480644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,63,0.029482667644818623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,255,0.06653333206971486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,255,0.08703999718030293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,511,0.08426666259765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,511,0.11673067013422649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,fp8,1023,0.14298133055369058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,4,128,1,float16,float16,1023,0.1106666624546051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,1,0.04867733518282572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,1,0.02258133391539256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,3,0.04834666848182678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,7,0.047968000173568726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,7,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,3,0.02260799954334895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,15,0.04825599988301595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,15,0.022682666778564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,63,0.022656001150608063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,63,0.04778666794300079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,31,0.04799466828505198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,31,0.022687998910744984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,127,0.049653331438700356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,127,0.02775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,255,0.041365332901477814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,255,0.05809600154558817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,511,0.07811200122038524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,511,0.056159997979799904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,float16,1023,0.12275200088818868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,128,8,128,1,float16,fp8,1023,0.08357866605122884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,1,0.43350398540496826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,1,0.33769067128499347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,3,0.44366931915283203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,3,0.3475840091705322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,7,0.4485280116399129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,31,0.5319413344065348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,15,0.468943993250529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,15,0.5291306575139364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,63,0.5351359844207764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,7,0.36208001772562665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,31,0.46817068258921307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,127,0.6394453446070353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,63,0.4699519872665405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,255,0.8675253391265869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,1,0.33882665634155273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,float16,255,0.9767306645711263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,1,128,1,float16,fp8,127,0.5620479981104533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,1,0.4335626761118571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,3,0.44305598735809326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,7,0.4478986660639445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,7,0.365941325823466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,3,0.34730132420857746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,31,0.5384533405303955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,15,0.531818668047587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,31,0.46987199783325195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,15,0.47096534570058185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,63,0.5433546702067057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,63,0.47235198815663654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,255,0.8725759983062744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,fp8,127,0.5623466571172079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,127,0.663152019182841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,1,0.06383466720581055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,2,128,1,float16,float16,255,1.0031999746958415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,1,0.05610666672388712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,3,0.06400533517201741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,3,0.056426664193471275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,7,0.06359466910362244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,7,0.05667200187842051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,15,0.06386666496594746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,31,0.06397866706053416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,15,0.05718400080998739
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,63,0.06470400094985962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,31,0.05947199960549673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,63,0.05686399837334951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,255,0.07698133091131847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,1,0.087909330924352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,float16,127,0.06567466755708058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,127,0.07268266876538594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,1,0.04844266672929128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,4,128,1,float16,fp8,255,0.10733333230018616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,3,0.08833600083986919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,3,0.0473333348830541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,7,0.08776533603668213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,15,0.08849599957466125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,7,0.0476693312327067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,31,0.0888266662756602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,15,0.04834666848182678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,31,0.0473333348830541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,63,0.08915199836095174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,63,0.046522667010625206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,127,0.09005866448084514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,127,0.06761600077152252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,float16,255,0.10478400190671285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,128,8,128,1,float16,fp8,255,0.0799786647160848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,1,0.8697386582692465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,1,0.6796853542327881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,3,0.8887786865234375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,3,0.6963679790496826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,7,0.8993546962738037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,15,0.9359253247578939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,7,0.7278506755828857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,31,0.9342506726582845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,31,1.074064016342163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,15,1.0622666676839192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,63,0.9415520032246908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,63,1.0891679922739665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,1,0.8693599700927734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,float16,127,1.2991946538289387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,1,0.6794346968332926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,1,128,1,float16,fp8,127,1.1286293665568035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,3,0.8984266916910807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,3,0.6994826793670654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,7,0.735765298207601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,7,0.9069973627726237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,15,1.0737760066986084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,15,0.9402026335398356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,63,1.1098240216573079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,31,0.9406293233235677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,31,1.0995360215504963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,63,0.9477972984313965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,fp8,127,1.1317439874013264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,1,0.12025599678357442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,2,128,1,float16,float16,127,1.3275466759999592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,1,0.17006933689117432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,7,0.12082667152086894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,3,0.12091733018557231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,3,0.1693120002746582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,7,0.16992533206939697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,15,0.12174933155377705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,15,0.170522669951121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,31,0.12230400244394939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,31,0.16996800899505615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,63,0.12214932839075725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,float16,127,0.12260799606641133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,127,0.1843679944674174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,4,128,1,float16,fp8,63,0.16924800475438437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,1,0.11342400312423706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,3,0.16740266482035318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,1,0.16664533813794455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,3,0.11314133803049724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,7,0.16821332772572836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,7,0.11329600214958191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,15,0.1135093371073405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,31,0.1697653333346049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,15,0.16937067111333212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,31,0.11346133550008138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,63,0.16876266400019327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,1,0.01863466699918111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,63,0.11356799801190694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,float16,127,0.16954133907953897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,128,8,128,1,float16,fp8,127,0.1292586624622345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,1,0.014560000350077948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,3,0.015205333630243937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,3,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,7,0.019274666905403137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,7,0.015583999454975128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,15,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,15,0.01951466624935468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,31,0.019519999623298645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,63,0.019461333751678467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,31,0.021327999730904896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,63,0.02162133405605952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,127,0.025258667767047882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,127,0.022821334501107533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,255,0.03648533423741659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,511,0.05819199979305267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,255,0.033200000723203026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,1023,0.1018986701965332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,511,0.05414933462937673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,1023,0.09598400195439656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,4095,0.36321067810058594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,2047,0.17969600359598795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,2047,0.18885332345962524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,4095,0.34569064776102704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,8191,0.7135199705759684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,16383,1.4126613934834797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,fp8,8191,0.6792266368865967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,1,128,1,float16,float16,16383,1.7644054094950359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,1,0.018757333358128864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,1,0.014767999450365702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,3,0.019237333287795384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,7,0.01934933289885521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,3,0.015146666516860327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,15,0.021488000949223835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,15,0.01939733326435089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,31,0.021375998854637146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,31,0.019424000134070713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,7,0.01579733317097028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,63,0.021583999196688335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,511,0.05417066812515259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,63,0.01950399950146675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,127,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,127,0.022863999009132385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,255,0.036288000643253326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,255,0.03329066683848699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,511,0.05821333328882853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,1023,0.10204799969991048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,2047,0.18947199980417886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,2047,0.17963733275731406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,1023,0.09605333209037781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,4095,0.3661706844965617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,4095,0.34541332721710205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,8191,0.6777439912160238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,8191,0.7925972938537598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,1,0.014639999717473984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,3,0.015189333508412043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,1,0.01893866683046023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,float16,16383,1.89683198928833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,2,128,1,float16,fp8,16383,1.4717334111531575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,3,0.01931200052301089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,7,0.01937066639463107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,15,0.02117866774400075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,7,0.015615999698638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,15,0.019589333484570186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,31,0.02165333429972331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,31,0.01934933289885521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,63,0.02184533327817917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,127,0.0252960001428922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,127,0.022917332748572033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,63,0.019386666516462963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,255,0.015311999867359797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,255,0.011920000116030375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,511,0.015840000162522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,1023,0.023418667415777843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,511,0.013440000514189402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,1023,0.025306666890780132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,2047,0.033786666889985405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,4095,0.05282666782538096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,4095,0.05346666773160299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,2047,0.03452266752719879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,8191,0.07462933162848155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,8191,0.09338666995366414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,float16,16383,0.11250133315722148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,1,0.010981333752473196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,3,0.007850666840871176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,1,0.007717333113153775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,4,128,1,float16,fp8,16383,0.13622400164604187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,3,0.01081066702802976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,15,0.008021333565314611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,7,0.007055999711155891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,31,0.010650667051474253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,7,0.01126933346192042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,31,0.008330666770537695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,63,0.008021333565314611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,63,0.012170666207869848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,127,0.010725333044926325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,127,0.007807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,255,0.01369599997997284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,511,0.011178666104873022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,255,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,511,0.019866666446129482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,1023,0.02864533414443334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,1023,0.019733333339293797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,4095,0.06881066660086314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,2047,0.04483200112978617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,2047,0.027855999767780304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,4095,0.04558933277924856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,8191,0.08841600020726521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,16383,0.08888000249862671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,float16,16383,0.13029332955678305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,128,8,128,1,float16,fp8,8191,0.061664000153541565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,1,1.7859360376993816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,1,1.3820053736368816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,3,1.8248426119486492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,7,1.8422452608744304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,7,1.4748586018880208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,3,1.424623966217041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,15,2.1372639338175454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,15,1.8735626538594563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,31,2.1514453887939453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,31,1.870314598083496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,float16,63,2.1711732546488443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,1,128,1,float16,fp8,63,1.8814239501953125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,3,1.8310720125834148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,1,1.7903626759847004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,3,1.4251999855041504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,7,1.4881493250528972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,1,1.3853599230448406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,7,1.845263957977295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,15,2.1521387100219727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,15,1.881717363993327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,63,2.21234130859375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,31,1.8788960774739583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,1,0.3333280086517334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,float16,31,2.1936052640279136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,2,128,1,float16,fp8,63,1.8903573354085286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,1,0.23348265886306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,3,0.23315733671188354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,3,0.33346132437388104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,7,0.23334399859110513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,7,0.3334346612294515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,15,0.23344000180562338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,15,0.3325173258781433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,63,0.23135999838511148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,float16,31,0.2337119976679484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,31,0.33373868465423584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,4,128,1,float16,fp8,63,0.3333653211593628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,1,0.3295946717262268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,3,0.21676266193389893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,3,0.3301386634508769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,1,0.21714133024215698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,7,0.2173866629600525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,7,0.3309280077616374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,15,0.21689067284266153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,15,0.3296746611595154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,31,0.3295466701189677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,31,0.2164426644643148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,fp8,63,0.21665600935618082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,128,8,128,1,float16,float16,63,0.32765867312749225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,3,2.84172789255778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,1,3.56493345896403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,7,3.679978688557943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,7,2.9434827168782554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,1,2.7586240768432617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,3,3.6432692209879556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,15,4.2688798904418945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,15,3.740074793497721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,1,3.5747200647989907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,float16,31,4.2951094309488935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,1,128,1,float16,fp8,31,3.7338291803995767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,3,2.8449014027913413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,1,2.7649332682291665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,3,3.653994560241699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,7,3.6818294525146484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,7,2.9707679748535156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,15,4.297381401062012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,float16,31,4.381157239278157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,1,0.657034675280253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,15,3.7568000157674155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,2,128,1,float16,fp8,31,3.7509333292643228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,1,0.4529333511988322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,3,0.4532800118128459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,3,0.6560746828715006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,7,0.4533653259277344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,7,0.6574506759643555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,31,0.6575466791788737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,fp8,15,0.6570826768875122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,31,0.4524159828821818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,1,0.6503946781158447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,4,128,1,float16,float16,15,0.4527680079142253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,1,0.42747199535369873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,3,0.6481546560923258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,3,0.42693865299224854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,7,0.6475893259048462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,7,0.42690666516621906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,15,0.6505013306935629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,31,0.4262186686197917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,1,0.033728001018365227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,float16,31,0.6478559970855713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,1,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,3,0.034373333056767784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,3,0.02651199946800868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,128,8,128,1,float16,fp8,15,0.42688532670338947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,7,0.034448000291983284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,15,0.037952000896135964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,7,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,15,0.03446399917205175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,31,0.03860266755024592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,127,0.04599999884764353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,31,0.03453866640726725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,63,0.03453333427508672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,63,0.038890667259693146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,127,0.04074133435885111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,511,0.11036266883214314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,255,0.06755200028419495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,255,0.06067200005054474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,1023,0.19568000237147012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,511,0.10186133782068889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,1023,0.18409067392349243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,2047,0.3667999903361003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,2047,0.3482453425725301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,8191,1.4321014086405437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,float16,4095,0.7090453306833903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,4095,0.6733012994130453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,1,128,1,float16,fp8,8191,1.3247040112813313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,1,0.033557333052158356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,1,0.025807999074459076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,3,0.034160000582536064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,3,0.026421333352724712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,7,0.027162666122118633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,31,0.034416000048319496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,15,0.03449599941571554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,15,0.03808533400297165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,31,0.03885333240032196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,7,0.034346667428811394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,127,0.04633066554864248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,63,0.038917332887649536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,63,0.034602666894594826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,127,0.04067733387152354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,255,0.0679253339767456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,511,0.10196266571680705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,255,0.061008001367251076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,1023,0.1967680056889852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,1023,0.18428266048431396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,2047,0.36840001742045086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,2047,0.34827733039855957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,511,0.11054399609565735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,4095,0.7483413219451904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,4095,0.6734240055084229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,float16,8191,1.473632017771403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,1,0.01268799975514412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,1,0.009232000137368837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,2,128,1,float16,fp8,8191,1.3301546573638916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,3,0.00938666673998038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,3,0.012357333054145178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,7,0.009306666751702627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,7,0.01239466667175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,31,0.00978133330742518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,15,0.009226666763424873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,31,0.012538666526476542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,15,0.01251199965675672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,63,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,127,0.010079999764760336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,63,0.009114666531483332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,127,0.01232533281048139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,255,0.016506666938463848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,255,0.01210133358836174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,511,0.023749334116776783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,1023,0.03422933320204417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,511,0.022661333282788593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,1023,0.04163199911514918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,2047,0.055215999484062195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,2047,0.05923733115196228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,4095,0.08505599697430928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,4095,0.11151466766993205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,float16,8191,0.11035199960072835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,1,0.010970667004585266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,4,128,1,float16,fp8,8191,0.13782933354377747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,1,0.007370666911204656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,3,0.011183999478816986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,3,0.00726400005320708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,7,0.007989333321650824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,7,0.01116266722480456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,15,0.01109333336353302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,31,0.011215999722480774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,31,0.007333333293596904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,63,0.00766933336853981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,63,0.01098666712641716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,15,0.010346666599313417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,127,0.011296000331640244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,127,0.00897066667675972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,511,0.028949332733949024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,255,0.021695998807748158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,511,0.01882133384545644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,1023,0.03305600086847941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,2047,0.0673280010620753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,1023,0.047557334105173744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,2047,0.0450133333603541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,4095,0.08731200297673543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,4095,0.06060799956321716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,float16,8191,0.1276853382587433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,1,0.044922664761543274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,128,8,128,1,float16,fp8,8191,0.08778132994969685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,1,0.036159999668598175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,3,0.045935998360315956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,3,0.03770666569471359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,7,0.04677333434422811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,7,0.03916800022125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,31,0.04924266537030538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,15,0.04921066761016846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,15,0.05475200215975443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,31,0.05550399919350942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,127,0.05829333265622457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,63,0.055904000997543335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,63,0.04944533109664917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,127,0.06619733572006226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,255,0.09834667046864827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,511,0.14970133701960245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,255,0.08851733803749084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,511,0.16150933504104614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,1023,0.2890506585439046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,1023,0.27224000295003253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,2047,0.5420159896214803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,2047,0.5162293513615926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,float16,4095,1.0754666328430176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,1,128,1,float16,fp8,4095,1.0001866817474365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,3,0.045909335215886436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,1,0.03660800059636434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,7,0.039173332353432976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,1,0.044821331898371376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,3,0.03770666569471359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,7,0.046666666865348816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,31,0.04942933221658071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,15,0.04940799872080485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,15,0.05481066803137461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,31,0.05548266569773356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,63,0.04962133367856344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,63,0.055871998270352684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,127,0.06643199920654297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,127,0.058448001742362976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,255,0.0988106628259023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,255,0.0886346697807312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,2047,0.5653226772944132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,1023,0.2723733385403951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,511,0.16220800081888834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,1023,0.2897866765658061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,511,0.14969600240389505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,2047,0.5166560014088949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,fp8,4095,1.0039947032928467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,2,128,1,float16,float16,4095,1.0958986282348633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,1,0.013045333325862885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,7,0.012975999464591345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,3,0.012970666090647379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,31,0.01303999995191892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,15,0.013093333691358566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,63,0.01310933381319046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,127,0.012938667088747025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,255,0.025008000433444977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,511,0.03425066669782003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,2047,0.07967466612656911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,1023,0.05672533313433329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,4,128,1,float16,float16,4095,0.10681066910425822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,1,0.015909332782030106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,1,0.007711999739209811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,3,0.015829333414634068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,3,0.008069333309928576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,7,0.016085332880417507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,7,0.007653333246707916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,15,0.01597333326935768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,15,0.007711999739209811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,31,0.01594666639963786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,31,0.007514666765928268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,63,0.01613866661985715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,63,0.007727999861041705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,127,0.016154666741689045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,127,0.0085333331177632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,255,0.030581332743167877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,255,0.01714666684468587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,511,0.047509332497914634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,511,0.03028800090154012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,1023,0.061994666854540505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,1023,0.041493333876132965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,2047,0.08578667044639587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,2047,0.056885331869125366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,float16,4095,0.12653332948684692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,96,8,128,1,float16,fp8,4095,0.08426666259765625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,1,0.008581333483258883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,1,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,3,0.008506666868925095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,3,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,7,0.008565333361426989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,7,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,15,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,15,0.009754666437705358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,31,0.009493333597977957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,31,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,63,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,63,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,127,0.00956266683836778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,127,0.012181332955757776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,255,0.010581333190202713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,255,0.01227733368674914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,511,0.027952000498771667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,511,0.027653334041436512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,1023,0.03239466746648153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,1023,0.03233599911133448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,2047,0.04373333354791006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,2047,0.04337066908677419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,4095,0.0658186674118042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,4095,0.0647573322057724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,8191,0.10960533221562703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,8191,0.10716799894968669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,float16,16383,0.1976319948832194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,1,128,1,float16,fp8,16383,0.19220799207687378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,1,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,3,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,3,0.009472000102202097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,7,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,7,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,15,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,15,0.009754666437705358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,31,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,31,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,63,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,63,0.012298667182525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,127,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,127,0.012186666329701742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,255,0.010640000303586325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,255,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,511,0.02777066578467687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,511,0.027850667635599773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,1023,0.03236266722281774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,1023,0.0322773332397143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,2047,0.043162668744723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,2047,0.043706665436426796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,4095,0.06588266789913177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,4095,0.06477866570154826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,8191,0.10969600081443787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,8191,0.1070186694463094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,float16,16383,0.19750400384267172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,2,128,1,float16,fp8,16383,0.19192532698313394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,1,0.008506666868925095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,3,0.00850133349498113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,7,0.01166933278242747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,15,0.008661333471536636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,31,0.009178666397929192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,63,0.009322666873534521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,127,0.009434666484594345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,255,0.010656000425418219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,511,0.027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,1023,0.032458665470282234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,2047,0.01716800034046173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,4095,0.018229333062966663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,8191,0.02310933421055476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,4,128,1,float16,float16,16383,0.025434667865435284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,1,0.008447999755541483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,3,0.009610666582981745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,1,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,3,0.009418666362762451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,7,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,15,0.009285333255926767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,31,0.009402666861812273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,31,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,63,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,7,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,63,0.012304000556468964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,127,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,255,0.012357333054145178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,127,0.012085333466529846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,255,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,511,0.0278613343834877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,511,0.02759466568628947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,1023,0.013898666948080063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,1023,0.009935999910036722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,2047,0.015504000087579092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,2047,0.011333333949247995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,4095,0.013093333691358566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,8191,0.025685332715511322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,16383,0.030661332110563915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,1,0.013722666849692663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,16383,0.022384000321229298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,1,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,3,0.013642666240533194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,3,0.009056000038981438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,7,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,7,0.0136266661187013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,15,0.01394133393963178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,fp8,8191,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,15,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,96,8,128,1,float16,float16,4095,0.016085332880417507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,31,0.015466666469971338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,31,0.011589333415031433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,63,0.011685332904259363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,127,0.011626667032639185
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,127,0.015791999797026317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,63,0.01551466683546702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,255,0.017781333376963932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,511,0.024069334069887798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,1023,0.03605866680542628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,511,0.018602666755517323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,2047,0.06062933305899302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,1023,0.029093332588672638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,4095,0.1090613305568695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,4095,0.09140266974767049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,8191,0.206112007300059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,8191,0.17407999436060587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,float16,16383,0.4002986749013265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,16383,0.3390666643778483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,1,0.013573333621025085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,1,0.012874666601419449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,3,0.013637332866589228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,3,0.010538666198650995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,7,0.013605333864688873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,2047,0.050053333242734276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,1,128,1,float16,fp8,255,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,7,0.00898133342464765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,15,0.01413333291808764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,15,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,31,0.011727999895811081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,31,0.015360000232855478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,63,0.015493333339691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,127,0.015770666301250458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,63,0.011429333438475927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,127,0.01173866664369901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,255,0.01786133274435997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,255,0.013557333499193192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,511,0.024127999941507976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,511,0.018559999763965607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,1023,0.028912000358104706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,1023,0.036330667634805046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,2047,0.06075733403364817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,2047,0.0499946673711141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,4095,0.10916266838709514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,8191,0.20665067434310913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,4095,0.09147733449935913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,8191,0.17398399114608765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,fp8,16383,0.3387840191523234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,2,128,1,float16,float16,16383,0.40067732334136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,1,0.01360000049074491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,3,0.013669333110253016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,15,0.014005333185195923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,7,0.013701333353916803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,31,0.01540800059835116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,63,0.015573333948850632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,127,0.015861333658297855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,255,0.017973333597183228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,511,0.02386133372783661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,1023,0.015509333461523056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,2047,0.018165333817402523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,8191,0.0314026673634847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,4095,0.02011200040578842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,4,128,1,float16,float16,16383,0.03806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,1,0.013493333011865616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,3,0.013317332913478216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,1,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,3,0.008842666943868002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,7,0.013343999783198038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,15,0.013914667069911957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,15,0.009701333319147428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,63,0.015658666690190632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,31,0.011658667276302973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,63,0.013381333400805792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,127,0.01570133368174235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,127,0.011754666765530905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,255,0.017898666361967724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,7,0.00980266680320104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,255,0.01357866699496905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,511,0.013317332913478216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,511,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,1023,0.013898666948080063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,31,0.015322666615247726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,2047,0.012741333494583765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,1023,0.010005333150426546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,4095,0.023989332218964893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,8191,0.035088000198205314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,4095,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,2047,0.02072000006834666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,float16,16383,0.0537120004494985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,1,0.08604799707730611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,8191,0.02604266752799352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,96,8,128,1,float16,fp8,16383,0.03495999922355016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,3,0.08778132994969685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,1,0.06752533217271169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,3,0.06949866811434428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,7,0.08893332878748576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,31,0.10472533106803894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,31,0.09248000383377075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,63,0.10575466354688008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,127,0.12611732880274454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,63,0.09284800291061401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,127,0.11025066177050273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,255,0.18844266732533774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,511,0.3106186588605245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,511,0.2874613404273987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,255,0.1683893402417501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,1023,0.5592853228251139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,1023,0.5261280139287313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,2047,1.0761066277821858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,2047,1.0015839735666912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,15,0.09240000446637471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,1,0.08589866757392883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,float16,15,0.10322667161623637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,1,0.06752533217271169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,1,128,1,float16,fp8,7,0.0727893312772115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,3,0.08770666519800822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,3,0.0695360004901886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,7,0.07291733225186665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,7,0.08841066559155782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,15,0.10314666231473286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,15,0.09249066313107808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,31,0.09259733557701111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,31,0.10471999645233154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,63,0.09294399619102478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,127,0.1097920040289561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,255,0.18852800130844116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,127,0.12597333391507468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,511,0.28735466798146564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,255,0.16871466239293417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,63,0.10583466291427612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,511,0.31113600730895996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,2047,1.004325310389201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,fp8,1023,0.5263626575469971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,1023,0.5798506736755371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,2,128,1,float16,float16,2047,1.0901813507080078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,1,0.020389333367347717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,3,0.019695999721686046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,7,0.019999999552965164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,15,0.022298666338125866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,31,0.02239466706911723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,63,0.019679999599854153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,127,0.01930133377512296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,255,0.03724266588687897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,2047,0.10679466525713603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,511,0.05834666887919108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,4,128,1,float16,float16,1023,0.07635733485221863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,1,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,1,0.012133333832025528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,3,0.025861332813898723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,3,0.012357333054145178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,7,0.02571733295917511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,15,0.026149332523345947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,7,0.012186666329701742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,15,0.012373333175977072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,31,0.026250667870044708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,31,0.012122667084137598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,63,0.02606933315594991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,127,0.02609066665172577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,63,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,255,0.0513973335425059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,127,0.014442666123310724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,255,0.02994133283694585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,511,0.06565333406130473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,511,0.04297600189844767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,1023,0.0842026670773824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,1023,0.057461331288019814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,float16,2047,0.13019200166066489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,96,8,128,1,float16,fp8,2047,0.08501332998275757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,1,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,3,0.014671999961137772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,1,0.014682666709025701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,3,0.019343999524911244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,7,0.01916266605257988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,7,0.014778666198253632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,15,0.019797333826621372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,15,0.01462399959564209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,31,0.02181333303451538
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,31,0.01676799977819125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,63,0.022245332598686218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,63,0.016858667135238647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,127,0.022346665461858112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,127,0.016735999534527462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,255,0.02586666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,255,0.0194560003777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,511,0.035071998834609985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,511,0.026757332185904186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,1023,0.05355200171470642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,1023,0.041738669077555336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,2047,0.09058666229248047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,2047,0.07237333556016286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,4095,0.16429866353670755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,4095,0.13395733634630838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,8191,0.3115839958190918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,8191,0.2564799984296163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,float16,16383,0.6071093479792277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,1,128,1,float16,fp8,16383,0.5014026562372843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,1,0.019215999792019527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,1,0.014607999473810196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,3,0.019354666272799175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,3,0.014746667196353277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,7,0.019088000059127808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,7,0.014736000448465347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,15,0.019621333728233974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,15,0.014757333944241205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,31,0.016762666404247284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,31,0.021749332547187805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,63,0.016778666526079178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,63,0.022240000466505688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,127,0.022517333428064983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,127,0.01672533278663953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,255,0.02573866645495097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,255,0.019386666516462963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,511,0.03510399907827377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,511,0.026917333404223125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,1023,0.041738669077555336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,1023,0.053717335065205894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,2047,0.07241599758466084
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,2047,0.09073600172996521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,4095,0.16502933700879416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,4095,0.13381866614023843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,8191,0.31414933999379474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,float16,16383,0.7270987033843994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,8191,0.2560746669769287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,1,0.019088000059127808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,2,128,1,float16,fp8,16383,0.5012906789779663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,3,0.01922133316596349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,7,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,15,0.019744000087181728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,31,0.02164799968401591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,63,0.022250667214393616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,127,0.022357332209746044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,255,0.025775998830795288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,511,0.01488000030318896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,1023,0.015370666980743408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,2047,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,8191,0.04563733438650767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,4095,0.030495998760064442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,4,128,1,float16,float16,16383,0.06574399769306183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,1,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,1,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,3,0.01481066644191742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,3,0.019146667172511418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,7,0.019088000059127808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,7,0.01852799952030182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,15,0.0198186660806338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,31,0.02165333429972331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,31,0.016837333639462788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,63,0.022277332842350006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,63,0.016794666647911072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,127,0.022410665949185688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,255,0.0129120002190272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,127,0.017008000363906223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,255,0.008762666955590248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,511,0.013733333597580591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,15,0.014501333236694336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,511,0.009365333244204521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,1023,0.019658666104078293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,1023,0.011253333340088526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,2047,0.019333332777023315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,2047,0.028618666032950085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,4095,0.0233599990606308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,8191,0.05859733124574026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,8191,0.0365226666132609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,4095,0.04244266450405121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,float16,16383,0.08904533584912618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,96,8,128,1,float16,fp8,16383,0.06165333092212677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,1,0.16662933429082236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,1,0.13007466991742453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,3,0.1333440045515696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,3,0.17022399107615152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,7,0.14032533764839172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,15,0.20217067003250122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,7,0.171834667523702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,15,0.17968533436457315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,31,0.20469866196314493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,31,0.1794453263282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,63,0.20595200856526694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,63,0.18045334021250406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,127,0.24597332874933878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,127,0.21423999468485513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,255,0.3702400128046672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,255,0.33060266574223834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,511,0.6125066677729288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,511,0.5677333275477091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,float16,1023,1.125386635462443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,1,128,1,float16,fp8,1023,1.0435252984364827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,1,0.16645866632461548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,1,0.13023466865221658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,3,0.17006399234135947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,3,0.13351466258366904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,7,0.14074132839838663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,15,0.2023680011431376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,7,0.17138133446375528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,15,0.17985600233078003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,31,0.20520534118016562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,31,0.17956799268722534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,63,0.2066239913304647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,63,0.18066666523615518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,127,0.24650667111078897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,127,0.21420266230901083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,255,0.3710613250732422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,255,0.33053332567214966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,511,0.634714682896932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,511,0.567631999651591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,float16,1023,1.1387893358866374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,2,128,1,float16,fp8,1023,1.0481493473052979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,1,0.034645333886146545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,3,0.03458133339881897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,7,0.03454400102297465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,15,0.03451200077931086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,31,0.03477333237727483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,63,0.03414933383464813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,127,0.033914667864640556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,255,0.06448533137639363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,511,0.08142933249473572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,4,128,1,float16,float16,1023,0.1077280044555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,1,0.04691733419895172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,1,0.02125866711139679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,3,0.046709333856900535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,3,0.021183999876181286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,7,0.04699199895064036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,15,0.02146133283774058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,31,0.021295999487241108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,63,0.04664533336957296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,63,0.02126399924357732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,7,0.021333334346612293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,15,0.04692266881465912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,127,0.048895999789237976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,127,0.025424001117547352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,255,0.05574933191140493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,31,0.047498668233553566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,255,0.036917333801587425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,1023,0.11867733796437581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,1023,0.08009600142637889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,float16,511,0.07652799785137177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,96,8,128,1,float16,fp8,511,0.05219733218352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,1,0.3275146683057149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,1,0.25571199258168537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,7,0.3382986783981323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,3,0.33402132987976074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,3,0.2622239987055461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,15,0.3992319901784261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,7,0.2747039993604024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,15,0.35423465569814044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,31,0.353770653406779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,31,0.4040373166402181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,63,0.40678401788075763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,63,0.35555732250213623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,127,0.4869920015335083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,255,0.6526346604029337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,float16,255,0.7400053342183431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,1,128,1,float16,fp8,127,0.42261866728464764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,1,0.3269333243370056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,3,0.3341013193130493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,3,0.2611786723136902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,1,0.25574400027592975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,7,0.3375999927520752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,7,0.2760000030199687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,15,0.39953064918518066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,15,0.354533314704895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,31,0.40536534786224365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,31,0.35388267040252686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,63,0.4084533452987671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,127,0.49697601795196533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,63,0.3559093475341797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,127,0.4227253198623657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,fp8,255,0.654858668645223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,2,128,1,float16,float16,255,0.7594506740570068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,1,0.061679999033610024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,3,0.06229333579540253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,7,0.061664000153541565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,15,0.06195733447869619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,31,0.06150933106740316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,63,0.06215466558933258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,127,0.06392000118891399
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,1,0.08628799517949422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,4,128,1,float16,float16,255,0.07242666681607564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,3,0.08627733588218689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,1,0.0409706657131513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,3,0.040752001106739044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,7,0.08610133330027263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,7,0.04088533421357473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,15,0.08635200063387553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,15,0.03997333347797394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,31,0.08659199873606364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,63,0.08732799688975017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,63,0.03984000037113825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,31,0.040991999208927155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,127,0.08802133798599243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,127,0.060165335734685264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,float16,255,0.10087466239929199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,96,8,128,1,float16,fp8,255,0.07284266750017802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,1,0.6500159899393717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,1,0.5076959927876791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,3,0.664517323176066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,3,0.52073601881663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,7,0.6708532969156901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,7,0.543557325998942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,15,0.7941706975301107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,15,0.7033653259277344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,31,0.8065333366394043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,31,0.7024693489074707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,63,0.8188213507334391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,63,0.7085013389587402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,float16,127,0.9926559925079346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,1,128,1,float16,fp8,127,0.8453973134358724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,1,0.6490240097045898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,1,0.5078666607538859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,3,0.6636639833450317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,3,0.5194933414459229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,7,0.6701599756876627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,7,0.5453973213831583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,15,0.7971519629160563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,15,0.7045653661092123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,31,0.817962646484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,31,0.7046453158060709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,63,0.8372266292572021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,127,0.8527946472167969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,fp8,63,0.7117013136545817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,2,128,1,float16,float16,127,1.007695992787679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,1,0.1135093371073405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,3,0.11375466982523601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,7,0.11404266953468323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,15,0.11517332990964253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,31,0.11497599879900615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,63,0.11447466413180034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,4,128,1,float16,float16,127,0.1149173378944397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,1,0.15922133127848306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,3,0.1590506633122762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,1,0.09897067149480183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,3,0.09885332981745402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,7,0.16035733620325723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,7,0.09937600294748943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,15,0.16119466225306192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,31,0.1629706621170044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,15,0.09918933113416036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,63,0.16130133469899496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,63,0.09905067086219788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,float16,127,0.1622986694176992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,127,0.11354666948318481
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,1,0.01402666668097178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,1,0.013765333841244379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,3,0.014021333307027817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,3,0.013999999811251959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,96,8,128,1,float16,fp8,31,0.09869333108266194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,7,0.013584000368913015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,15,0.01657066618402799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,15,0.01562133307258288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,31,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,31,0.01597333326935768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,63,0.01661866654952367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,63,0.015856000284353893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,7,0.01452800010641416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,127,0.019808000574509304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,127,0.018160000443458557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,255,0.028021333118279774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,255,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,511,0.04491200049718221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,511,0.04199466605981191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,1023,0.07807999849319458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,1023,0.07362666726112366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,2047,0.14388799667358398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,4095,0.26366933186848956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,2047,0.1373546620210012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,4095,0.2752799987792969
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,8191,0.5387413501739502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,1,0.014021333307027817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,8191,0.5165919860204061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,fp8,16383,1.0677493413289387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,1,128,1,float16,float16,16383,1.4390560785929363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,3,0.014256000518798828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,1,0.013631999492645264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,3,0.013829333086808523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,7,0.014560000350077948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,7,0.013866666704416275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,15,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,15,0.01570133368174235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,31,0.01666133354107539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,63,0.016986666868130367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,63,0.015802666544914246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,127,0.01959466685851415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,127,0.01823466643691063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,255,0.026208000878492992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,511,0.04474133253097534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,255,0.027984000742435455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,511,0.04203199843565623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,1023,0.07376533250013988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,2047,0.14401599764823914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,31,0.015813333292802174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,2047,0.13724266489346823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,4095,0.2789600094159444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,4095,0.26333866516749066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,8191,0.5731840133666992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,16383,1.1254346370697021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,16383,1.463311990102132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,float16,1023,0.07765333354473114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,2,128,1,float16,fp8,8191,0.5153760115305582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,1,0.014117332796255747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,3,0.014042666802803675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,7,0.014245333770910898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,15,0.016586666305859882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,31,0.016447999825080235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,63,0.016783999900023144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,127,0.01953599974513054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,511,0.015397333850463232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,255,0.014970666418472925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,1023,0.022709332406520844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,2047,0.033488000432650246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,4095,0.05120533208052317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,8191,0.07326933244864146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,4,128,1,float16,float16,16383,0.10997333129247029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,1,0.01848000039656957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,1,0.008933333059151968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,3,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,3,0.006821333120266597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,7,0.01044800008336703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,7,0.006757333253820737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,15,0.010869332899649939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,15,0.006853333363930385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,31,0.01062400018175443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,31,0.007098666702707608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,63,0.010458666831254959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,63,0.00702400008837382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,255,0.009029333169261614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,255,0.013712000101804733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,1023,0.018965333700180054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,1023,0.028389332195123036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,2047,0.04385066529115041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,2047,0.0262719988822937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,4095,0.06717333197593689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,4095,0.04299733539422353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,127,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,127,0.007647999872763951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,8191,0.05858133236567179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,511,0.019173332800467808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,16383,0.12760532895723978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,float16,8191,0.08604799707730611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,16383,0.08536000053087871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,96,8,128,1,float16,fp8,511,0.010687999427318573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,1,1.3465760548909504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,1,1.044975996017456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,3,1.374608039855957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,7,1.3868799209594727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,7,1.1145866711934407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,3,1.075114647547404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,15,1.6080533663431804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,15,1.4125173886617024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,31,1.636122703552246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,31,1.4108799298604329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,fp8,63,1.4201919237772624
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,1,128,1,float16,float16,63,1.6508480707804363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,float16,1,1.3498719533284504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,float16,3,1.3787573178609211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,fp8,7,1.1235360304514568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,fp8,1,1.051701307296753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,float16,7,1.3910346031188965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,fp8,3,1.0765493710835774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,float16,15,1.6168425877888997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,fp8,15,1.4158239364624023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,float16,31,1.6495679219563801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,fp8,31,1.413973331451416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,float16,63,1.667909304300944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,2,128,1,float16,fp8,63,1.4230666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,1,0.21945067246754965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,3,0.2198773423830668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,7,0.21964800357818604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,15,0.21994133790334067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,31,0.2202613353729248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,4,128,1,float16,float16,63,0.21868799130121866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,1,0.31324267387390137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,1,0.18850133816401163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,3,0.313914676507314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,3,0.18861865997314453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,7,0.31413867076237995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,7,0.18905067443847656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,15,0.3150399923324585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,15,0.18927466869354248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,31,0.3153653343518575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,31,0.1885813275973002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,float16,63,0.312608003616333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,96,8,128,1,float16,fp8,63,0.18823999166488647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,1,2.688826560974121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,1,2.082896073659261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,3,2.745541254679362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,3,2.144282658894857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,7,2.7684319814046225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,7,2.2231040000915527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,15,3.2095680236816406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,15,2.8189547856648765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,float16,31,3.265061378479004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,1,128,1,float16,fp8,31,2.8148374557495117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,float16,1,2.696096102396647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,fp8,1,2.093850612640381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,fp8,3,2.1460320154825845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,float16,3,2.7547734578450522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,float16,15,3.22650146484375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,float16,7,2.7752211888631186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,fp8,31,2.8215627670288086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,fp8,7,2.240330696105957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,fp8,15,2.8250560760498047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,2,128,1,float16,float16,31,3.291802724202474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,3,0.4275253216425578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,7,0.42879998683929443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,31,0.42822933197021484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,15,0.42712001005808514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,4,128,1,float16,float16,1,0.42769066492716473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,1,0.618938684463501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,1,0.3670026858647664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,3,0.6198773384094238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,3,0.3668533166249593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,7,0.6189866860707601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,7,0.36775465806325275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,15,0.6198720137278239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,15,0.36777599652608234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,float16,31,0.6189493338267008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,96,8,128,1,float16,fp8,31,0.36642134189605713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,1,0.024218666056791942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,1,0.020309332758188248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,3,0.024613333245118458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,3,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,7,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,15,0.02958400050799052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,7,0.021429332594076794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,15,0.02693866689999898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,31,0.029834667841593426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,31,0.026933332284291584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,63,0.030042665700117748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,63,0.026949333647886913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,127,0.03557866563399633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,127,0.031744000812371574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,255,0.04698666433493296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,255,0.05193600058555603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,511,0.0843999981880188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,511,0.07797866563002269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,1023,0.1490133305390676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,1023,0.14030399918556213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,2047,0.27797333399454754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,2047,0.2640320062637329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,4095,0.5368320147196451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,4095,0.5096426804860433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,float16,8191,1.1033546924591064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,1,128,1,float16,fp8,8191,1.0024853547414143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,1,0.024362665911515553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,1,0.02027200038234393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,3,0.024746666351954143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,3,0.02351466566324234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,7,0.02149333308140437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,7,0.02532800038655599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,15,0.029653333127498627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,15,0.026943999032179516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,31,0.029930666089057922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,31,0.026842666169007618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,63,0.029909332593282063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,63,0.02701866626739502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,127,0.03549866626660029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,127,0.03160533308982849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,255,0.05189866820971171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,255,0.04714666803677877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,511,0.08461866776148479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,511,0.07796266674995422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,1023,0.14945066968599954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,1023,0.1402720014254252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,2047,0.2794026732444763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,2047,0.2640106678009033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,4095,0.5747573375701904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,4095,0.5093386570612589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,float16,8191,1.1301013628641765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,2,128,1,float16,fp8,8191,1.007269303003947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,1,0.012458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,3,0.012309333930412928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,7,0.01209066684047381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,15,0.012309333930412928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,31,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,63,0.012138667205969492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,127,0.01190399999419848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,255,0.015754666179418564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,511,0.022863999009132385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,1023,0.033520000676314034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,2047,0.05384000142415365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,4095,0.08072000245253245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,4,128,1,float16,float16,8191,0.10827733079592387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,1,0.010682666053374609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,1,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,3,0.010869332899649939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,3,0.0075626665105422335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,7,0.01817600056529045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,7,0.00744000015159448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,15,0.01097600037852923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,15,0.007466666400432587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,31,0.010954666882753372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,31,0.0075040000180403394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,63,0.010970667004585266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,63,0.0074506668994824094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,127,0.010837333897749582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,127,0.008143999924262365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,255,0.02130666623512904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,255,0.0102186668664217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,511,0.02845333268245061
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,511,0.017946666727463405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,1023,0.04683733483155569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,1023,0.03142933299144109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,2047,0.06617600222428639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,2047,0.04130133241415024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,4095,0.0855519970258077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,4095,0.05792533357938131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,float16,8191,0.12666133046150208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,96,8,128,1,float16,fp8,8191,0.08513599634170532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,1,0.033717334270477295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,1,0.025631998976071674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,3,0.0342399999499321
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,3,0.02649066597223282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,7,0.03462400039037069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,7,0.027232001225153606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,15,0.038319999972979225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,15,0.03444266567627589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,31,0.03870933254559835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,63,0.03923200070858002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,63,0.03451200077931086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,127,0.04065066576004028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,255,0.06810666620731354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,31,0.03429866582155228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,255,0.0609440008799235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,511,0.10193066795667012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,127,0.046384001771608986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,511,0.11061333616574605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,1023,0.19610132773717245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,1023,0.18425067265828451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,2047,0.36664533615112305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,2047,0.3481333255767822
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,4095,0.7448480129241943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,4095,0.6729919910430908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,float16,8191,1.4641013145446777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,1,128,1,float16,fp8,8191,1.330074628194173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,1,0.012533333152532578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,3,0.012624000509579977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,7,0.01251199965675672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,3,0.010346666599313417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,1,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,15,0.012373333175977072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,31,0.010149333626031876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,7,0.0100853331387043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,63,0.012432000289360682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,63,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,31,0.012479999413092932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,127,0.012149333953857422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,255,0.015925332903862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,127,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,255,0.011946666985750198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,511,0.023445333043734234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,511,0.022522665560245514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,1023,0.03388266762097677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,1023,0.04165333261092504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,2047,0.054858664671579994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,2047,0.059157331784566246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,4095,0.08447466293970744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,4095,0.1063253382841746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,fp8,8191,0.1376426617304484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,2,128,1,float16,float16,8191,0.10873599847157796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,1,0.011322667201360067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,3,0.01121066634853681
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,1,0.00785600021481514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,7,0.011178666104873022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,7,0.00786666696270307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,15,0.011226666470368704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,15,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,3,0.00725333330531915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,31,0.011221333096424738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,31,0.007770666852593422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,63,0.011061333119869232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,127,0.008752000207702318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,127,0.01110400011142095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,63,0.007658666620651881
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,511,0.018944000204404194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,1023,0.04740799963474274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,2047,0.04529066880544027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,1023,0.03316266586383184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,2047,0.06671466430028279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,4095,0.08668266733487447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,4095,0.06054399907588959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,8191,0.08758933345476787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,8191,0.12829867005348206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,255,0.021520001192887623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,fp8,255,0.01062400018175443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,4,128,1,float16,float16,511,0.028688001135985058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,1,0.007290666922926903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,3,0.007130666946371396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,7,0.007184000064929326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,15,0.00706666645904382
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,31,0.007082666580875714
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,1,0.015850666910409927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,63,0.007146666447321574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,15,0.01580799991885821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,7,0.01589866727590561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,3,0.01588800052801768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,63,0.015626666446526844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,127,0.015829333414634068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,255,0.0305226668715477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,127,0.008389333263039589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,511,0.04736533264319102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,511,0.021967999637126923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,1023,0.06172266602516174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,1023,0.028698667883872986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,31,0.01562133307258288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,255,0.014896000425020853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,4095,0.12754666805267334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,8191,0.1130400002002716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,2047,0.08530132969220479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,float16,8191,0.21247466405232748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,1,0.008357333640257517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,3,0.010005333150426546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,2047,0.04307733476161957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,64,8,128,1,float16,fp8,4095,0.06696000198523204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,1,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,3,0.009450666606426239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,7,0.008421333506703377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,7,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,15,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,15,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,31,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,31,0.010277333358923594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,63,0.009493333597977957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,63,0.012085333466529846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,255,0.01062400018175443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,511,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,511,0.016949333250522614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,1023,0.029711998999118805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,1023,0.029445332785447437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,2047,0.037647999823093414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,4095,0.05108266572157542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,2047,0.03703466554482778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,4095,0.05213866631189982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,8191,0.07950933277606964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,8191,0.08170666793982188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,16383,0.1400106648604075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,16383,0.136272003253301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,float16,32767,0.2574186722437541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,1,0.008512000242869059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,1,128,1,float16,fp8,32767,0.24940266211827597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,7,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,15,0.008912000184257826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,15,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,1,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,3,0.00842666688064734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,31,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,3,0.009365333244204521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,7,0.008432000254591307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,63,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,63,0.012175999581813812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,127,0.012096000214417776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,127,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,255,0.010698666175206503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,255,0.012133333832025528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,511,0.016794666647911072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,1023,0.02942399928967158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,511,0.01682666689157486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,2047,0.03774400055408478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,2047,0.03696533292531967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,8191,0.023071999351183575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,8191,0.022229333718617756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,1023,0.029711998999118805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,16383,0.024282666544119518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,4095,0.018309333672126133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,16383,0.026549334327379864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,float16,32767,0.03130666663249334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,4095,0.018415999909241993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,1,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,2,128,1,float16,fp8,32767,0.034304000437259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,3,0.009541333342591921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,7,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,7,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,15,0.008725333337982496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,15,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,1,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,31,0.011584000041087469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,31,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,3,0.008336000144481659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,255,0.011717333147923151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,127,0.009381333366036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,127,0.012191999703645706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,511,0.016810666769742966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,255,0.012096000214417776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,511,0.016672000288963318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,63,0.012138667205969492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,63,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,2047,0.015306666493415833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,1023,0.029680001238981884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,1023,0.029552000264326733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,2047,0.011744000017642975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,4095,0.015743999431530636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,4095,0.013381333400805792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,8191,0.019386666516462963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,16383,0.02199466774861018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,16383,0.0195573332409064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,32767,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,float16,32767,0.03522133330504099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,7,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,1,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,1,0.00956266683836778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,4,128,1,float16,fp8,8191,0.016058667252461117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,3,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,3,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,7,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,15,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,15,0.009877333417534828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,31,0.009258666386206945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,127,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,63,0.009648000200589498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,63,0.012282667060693106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,127,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,511,0.016869333883126576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,255,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,255,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,31,0.01033599985142549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,2047,0.015376000354687372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,511,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,1023,0.013546666751305262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,1023,0.00855466661353906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,8191,0.025642665723959606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,2047,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,4095,0.01670933390657107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,4095,0.011029332876205444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,8191,0.014677333335081736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,16383,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,32767,0.05040533343950907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,float16,16383,0.030741333961486816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,64,8,128,1,float16,fp8,32767,0.026549334327379864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,1,0.009205333267649015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,7,0.008517333616813024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,1,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,3,0.00867733359336853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,3,0.009525333220760027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,7,0.010117333382368088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,31,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,15,0.008778666456540426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,31,0.010474666953086853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,15,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,127,0.012378666549921036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,63,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,63,0.01232533281048139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,127,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,511,0.02794133375088374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,255,0.010656000425418219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,255,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,511,0.028037334481875103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,2047,0.04369066655635834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,1023,0.03250666707754135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,1023,0.03258133431275686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,2047,0.04399999976158142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,4095,0.06610133250554402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,8191,0.10777067144711812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,4095,0.06520000100135803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,8191,0.10987733801205952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,1,0.008986666798591614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,16383,0.19806933403015137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,16383,0.19275200366973877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,float16,32767,0.37380266189575195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,1,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,1,128,1,float16,fp8,32767,0.36213866869608563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,3,0.009754666437705358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,3,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,7,0.00847999999920527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,31,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,7,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,15,0.008992000172535578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,15,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,31,0.009445333232482275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,127,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,63,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,63,0.012341332932313284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,255,0.010773333410422007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,127,0.012527999778588613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,511,0.028037334481875103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,255,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,511,0.028234665592511494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,1023,0.032698666055997215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,4095,0.019343999524911244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,1023,0.032485333581765495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,2047,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,2047,0.016037333756685257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,4095,0.018016000588734944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,16383,0.026575999955336254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,8191,0.023957334458827972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,8191,0.022384000321229298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,16383,0.0268053337931633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,1,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,1,0.008400000010927519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,float16,32767,0.045408000548680626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,2,128,1,float16,fp8,32767,0.049770668148994446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,3,0.0084906667470932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,7,0.009599999835093817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,3,0.00973866693675518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,7,0.008586666857202848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,15,0.008858666444818178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,63,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,15,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,31,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,31,0.010496000448862711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,63,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,255,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,127,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,127,0.012149333953857422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,255,0.012357333054145178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,511,0.02829866607983907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,1023,0.0102186668664217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,511,0.02792533238728841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,1023,0.013877333452304205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,2047,0.016143999993801117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,8191,0.026416001220544178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,2047,0.011882666498422623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,4095,0.017136000096797943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,4095,0.013568000247081121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,16383,0.03302400062481562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,8191,0.018677332748969395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,float16,32767,0.05273599922657013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,16383,0.022778667509555817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,4,128,1,float16,fp8,32767,0.0422986646493276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,7,0.008639999975760778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,1,0.008453333129485449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,1,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,3,0.008549333239595095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,3,0.009701333319147428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,15,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,7,0.009701333319147428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,15,0.008842666943868002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,31,0.009493333597977957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,127,0.009733333562811216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,31,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,63,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,63,0.012282667060693106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,127,0.0122079998254776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,255,0.010693332801262537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,511,0.012981332838535309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,255,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,4095,0.023984000086784363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,511,0.008101333553592363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,1023,0.013712000101804733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,2047,0.02083733429511388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,1023,0.009039999917149544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,2047,0.010928000013033548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,8191,0.0353973334034284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,4095,0.012730666746695837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,8191,0.017845333864291508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,16383,0.05412266651789347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,1,0.06000000238418579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,16383,0.029365333418051403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,float16,32767,0.08145600060621898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,64,8,128,1,float16,fp8,32767,0.043466667334238686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,1,0.04653333127498627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,7,0.061717331409454346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,3,0.06113066772619883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,3,0.04808533191680908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,7,0.05056533217430115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,15,0.06367466847101848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,15,0.07123733560244243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,31,0.0636053333679835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,31,0.0718560020128886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,255,0.11590933799743652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,63,0.07256000240643819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,63,0.0639573335647583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,127,0.0860533316930135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,127,0.07589333256085713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,255,0.12864533066749573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,511,0.19708800315856934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,511,0.21213332811991373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,1023,0.3815466562906901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,1023,0.3600800037384033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,2047,0.7321920394897461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,2047,0.6836000283559164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,fp8,4095,1.3288959662119548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,1,128,1,float16,float16,4095,1.4150932629903157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,1,0.013242666920026144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,7,0.009418666362762451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,1,0.009381333366036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,3,0.013056000073750814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,3,0.009434666484594345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,7,0.013338666409254074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,15,0.013151999562978745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,31,0.013199999928474426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,31,0.009450666606426239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,63,0.01312000056107839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,255,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,63,0.009237333511312803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,127,0.01314666618903478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,127,0.009989333028594652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,511,0.040063999593257904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,15,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,255,0.021365332106749218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,511,0.035391998787721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,1023,0.05807999769846598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,1023,0.07524266839027405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,2047,0.11124799648920695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,2047,0.08483200271924336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,3,0.008026666939258575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,float16,4095,0.10921600461006165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,1,0.016741332908471424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,2,128,1,float16,fp8,4095,0.1388159990310669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,1,0.007786666974425316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,3,0.0163680004576842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,7,0.016442666451136272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,7,0.007850666840871176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,15,0.016352000335852306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,15,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,31,0.01634666696190834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,31,0.007791999727487564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,63,0.016106666376193363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,127,0.016261332978804905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,63,0.007754666730761528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,1023,0.06372266511122386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,127,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,255,0.03173866619666418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,255,0.018197332819302876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,511,0.032111999889214836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,511,0.04885333279768626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,1023,0.04534400006135305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,2047,0.08728532989819844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,2047,0.06032533446947733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,1,0.026026666164398193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,fp8,4095,0.08772800366083781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,1,0.010458666831254959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,4,128,1,float16,float16,4095,0.13030399878819784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,3,0.025983999172846477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,3,0.011674666156371435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,7,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,7,0.026026666164398193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,15,0.026101333399613697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,15,0.010431999961535135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,31,0.026629333694775898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,63,0.025637333591779072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,31,0.010485333700974783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,511,0.06515733400980632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,127,0.012266666938861212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,127,0.02625600000222524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,255,0.05093866586685181
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,255,0.021967999637126923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,63,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,511,0.030271999537944794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,1023,0.08400533596674602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,1023,0.04334933559099833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,2047,0.12948800126711527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,1,0.013770667215188345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,2047,0.06709333260854085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,float16,4095,0.2168160080909729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,1,0.009952000031868616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,64,8,128,1,float16,fp8,4095,0.11433600385983785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,3,0.01394133393963178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,3,0.008986666798591614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,7,0.013888000200192133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,15,0.014282666146755219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,7,0.009061333412925402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,15,0.010709332923094431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,31,0.011695999652147293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,63,0.016074666132529575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,63,0.011605333536863327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,127,0.016176000237464905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,127,0.01173866664369901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,255,0.01834133391578992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,511,0.018842666099468868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,31,0.015824000040690105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,255,0.01349866638580958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,511,0.02454400062561035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,1023,0.03718933214743932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,2047,0.06182933350404104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,1023,0.029359998802344005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,2047,0.050053333242734276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,4095,0.09161600470542908
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,8191,0.21020267407099405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,8191,0.17403199275334677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,16383,0.40721599260965985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,16383,0.3392159938812256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,4095,0.1109279990196228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,3,0.013962666193644205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,float16,32767,0.970362663269043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,1,0.01392000044385592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,1,0.009008000294367472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,1,128,1,float16,fp8,32767,0.7036426862080892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,3,0.00915733352303505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,7,0.010197333370645842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,15,0.014346666634082794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,15,0.009914666414260864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,31,0.011994666109482447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,31,0.015674666812022526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,63,0.01594666639963786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,7,0.013898666948080063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,63,0.011839999506870905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,127,0.011941333611806234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,127,0.016074666132529575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,255,0.018378666291634243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,255,0.01341333364446958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,1023,0.015749332805474598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,511,0.024586667617162068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,511,0.018768000106016796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,1023,0.013978666315476099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,2047,0.0199946661790212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,4095,0.0183999997874101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,2047,0.01573333392540614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,4095,0.02204799900452296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,8191,0.0332640012105306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,32767,0.06698133548100789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,float16,16383,0.04038399954636892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,16383,0.0402399996916453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,1,0.014042666802803675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,32767,0.07041599849859874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,1,0.008965333302815756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,2,128,1,float16,fp8,8191,0.032773333291212715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,3,0.013877333452304205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,3,0.009114666531483332
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,7,0.013823999712864557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,7,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,15,0.014325333138306936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,15,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,31,0.012159999459981918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,127,0.011792000383138657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,31,0.015605332950750986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,63,0.016000000139077503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,63,0.011866666376590729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,127,0.016042667130629223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,255,0.013514666507641474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,255,0.018378666291634243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,511,0.013183999806642532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,1023,0.013594667116800943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,2047,0.013280000537633896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,2047,0.022416000564893086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,1023,0.010543999572594961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,511,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,4095,0.025466665625572205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,4095,0.015589332828919092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,8191,0.02717333287000656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,8191,0.03792533278465271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,16383,0.05708266794681549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,16383,0.03488533447186152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,1,0.014010666559139887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,fp8,32767,0.062128002444903054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,4,128,1,float16,float16,32767,0.08546132842699687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,1,0.009984000275532404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,3,0.013978666315476099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,3,0.009093333035707474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,7,0.013951999445756277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,31,0.015487999965747198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,7,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,15,0.014149333039919535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,31,0.012042666474978128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,63,0.016085332880417507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,63,0.011871999750534693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,127,0.016058667252461117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,127,0.01190399999419848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,255,0.012757333616415659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,511,0.013434667140245438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,511,0.008341333518425623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,255,0.009098666409651438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,1023,0.01907733331123988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,1023,0.011045332998037338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,2047,0.028538666665554047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,2047,0.012533333152532578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,4095,0.041722665230433144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,16383,0.08919999996821086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,4095,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,8191,0.02700799951950709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,8191,0.059343998630841575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,1,0.11285866300264995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,16383,0.0473280002673467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,float16,32767,0.13007466991742453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,64,8,128,1,float16,fp8,32767,0.06972800195217133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,1,0.0888266662756602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,3,0.11516799529393514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,7,0.11708266536394756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,3,0.09103467067082723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,7,0.09548800190289815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,15,0.13706666231155396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,31,0.13892799615859985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,15,0.12184533476829529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,63,0.1400213340918223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,31,0.12151466806729634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,63,0.12214932839075725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,511,0.41252267360687256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,127,0.16693333784739176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,127,0.14484266440073648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,255,0.24952532847722372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,255,0.2230986754099528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,511,0.3824319839477539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,1023,0.7018773555755615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,1023,0.760095993677775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,1,0.02091199904680252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,float16,2047,1.4254080454508464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,1,0.016117333124081295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,3,0.02048533285657565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,1,128,1,float16,fp8,2047,1.3393227259318035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,3,0.016149333367745083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,7,0.016261332978804905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,15,0.020714666694402695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,63,0.020053333292404812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,31,0.020629333953062694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,7,0.020175999651352566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,31,0.016250666230916977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,15,0.01613866661985715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,127,0.017690667261679966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,63,0.016069332758585613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,127,0.020693333198626835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,255,0.03940266619126002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,255,0.0390079990029335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,1023,0.07849599917729695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,511,0.060362666845321655
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,1023,0.11251733700434367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,511,0.0784800002972285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,float16,2047,0.10958400368690491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,2,128,1,float16,fp8,2047,0.13953066865603128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,1,0.02735999971628189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,1,0.012991999586423239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,3,0.013061333447694778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,7,0.012831999609867731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,3,0.027327999472618103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,31,0.027306665976842243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,15,0.027386667827765148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,7,0.027466667195161183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,15,0.013082666943470636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,31,0.013013333082199097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,63,0.0273333340883255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,127,0.014933332800865173
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,255,0.0320266659061114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,63,0.012986666212479273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,255,0.05268266797065735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,127,0.027210667729377747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,511,0.06778133412202199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,511,0.04688533147176107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,1023,0.06117333471775055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,1023,0.0865119993686676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,float16,2047,0.1336373289426168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,4,128,1,float16,fp8,2047,0.0885653297106425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,1,0.014975999792416891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,1,0.04587199787298838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,3,0.04603200157483419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,7,0.04621333380540212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,15,0.014943999548753103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,7,0.014970666418472925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,15,0.04595200220743815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,31,0.014970666418472925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,31,0.04609066744645437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,63,0.045519997676213585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,63,0.015082667271296183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,127,0.04785599807898203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,255,0.02622399975856145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,511,0.04002666721741358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,127,0.01806933308641116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,255,0.055306668082873024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,511,0.07478933533032735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,1023,0.11859200398127238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,1023,0.06397866706053416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,float16,2047,0.20815465847651163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,64,8,128,1,float16,fp8,2047,0.11082667112350464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,3,0.22535467147827148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,1,0.2205386757850647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,3,0.17680533727010092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,1,0.17255999644597372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,7,0.22897066672643027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,7,0.18600000937779745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,15,0.2687573234240214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,15,0.2384106715520223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,31,0.27237866322199505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,31,0.23789334297180176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,63,0.2748639980951945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,63,0.23941334088643393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,127,0.3286133408546448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,127,0.28385066986083984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,255,0.4378453493118286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,255,0.4951466719309489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,float16,511,0.8338932991027832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,1,0.03536533315976461
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,1,128,1,float16,fp8,511,0.7539573510487875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,1,0.029311999678611755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,3,0.035189333061377205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,3,0.029322666426499683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,7,0.03501333296298981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,7,0.029333333174387615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,15,0.035189333061377205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,15,0.029306667546431225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,31,0.03495466709136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,31,0.029306667546431225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,63,0.03482666611671448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,63,0.02923733244339625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,127,0.03513599932193756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,127,0.0324799989660581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,255,0.06706133484840393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,255,0.08904000123341878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,fp8,511,0.11659199992815654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,1,0.04809066653251648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,2,128,1,float16,float16,511,0.08375466863314311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,1,0.022629333039124806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,3,0.048842668533325195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,7,0.04837866624196371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,7,0.022613334159056347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,15,0.048010667165120445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,3,0.022639999787012737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,15,0.022730665902296703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,31,0.048453330993652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,31,0.022704000274340313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,63,0.048170665899912514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,63,0.022682666778564453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,127,0.02749866743882497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,127,0.04966400067011515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,255,0.04142399877309799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,255,0.058373332023620605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,float16,511,0.0776800016562144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,4,128,1,float16,fp8,511,0.05622399846712748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,1,0.08383466800053914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,1,0.02587733417749405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,3,0.026517334083716076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,7,0.08354666829109192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,7,0.02590399980545044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,3,0.08359466989835103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,15,0.08389332890510559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,31,0.08380800485610962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,31,0.026399999856948853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,63,0.02590399980545044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,15,0.025968000292778015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,63,0.08458133538564046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,127,0.03638399889071783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,255,0.09869866569836934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,255,0.04930133124192556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,127,0.08589866757392883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,float16,511,0.13351466258366904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,1,0.43537068367004395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,64,8,128,1,float16,fp8,511,0.0699786643187205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,1,0.34120531876881915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,3,0.44517334302266437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,3,0.34966933727264404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,15,0.5326666831970215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,7,0.36613865693410236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,15,0.47141865889231366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,7,0.4517600138982137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,31,0.5399146477381388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,31,0.47051199277242023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,63,0.5466506481170654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,63,0.47365331649780273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,127,0.6658613284428915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,127,0.5620053211847941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,float16,255,1.0048213005065918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,1,0.0631466656923294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,1,0.05725333094596863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,1,128,1,float16,fp8,255,0.8739039897918701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,3,0.056261335810025535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,3,0.06368533273537953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,7,0.06363733112812042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,15,0.06339733302593231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,7,0.05750399827957153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,15,0.05753066639105479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,31,0.06422933439413707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,31,0.056426664193471275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,63,0.056015998125076294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,63,0.06426666676998138
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,127,0.06569066643714905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,float16,255,0.07514666517575581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,127,0.07462400197982788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,1,0.0879253347714742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,3,0.08840533097585042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,2,128,1,float16,fp8,255,0.10723732908566792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,1,0.048709332942962646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,3,0.04797333478927612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,7,0.08794132868448894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,7,0.047930667797724404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,31,0.08923199772834778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,15,0.046800002455711365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,15,0.08844266335169475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,31,0.04877333343029022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,63,0.08950933814048767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,63,0.04753600060939789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,127,0.06765333314736684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,127,0.09033600489298503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,fp8,255,0.08029333253701527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,4,128,1,float16,float16,255,0.1053013304869334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,1,0.15620799859364828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,1,0.05721066892147064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,3,0.15610667069753012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,15,0.15712533394495645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,3,0.05754133562246958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,7,0.05764266848564148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,15,0.05778666834036509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,7,0.1562933325767517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,31,0.15982400377591452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,31,0.05793066819508871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,63,0.16061866283416748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,63,0.05731200178464254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,127,0.16145599881807962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,127,0.06765866776307423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,fp8,255,0.08733866612116496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,1,0.01488000030318896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,1,0.013525333255529404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,64,8,128,1,float16,float16,255,0.18490666151046753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,3,0.013893333574136099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,3,0.014896000425020853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,15,0.014229333649079004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,7,0.015178666760524115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,15,0.01626666635274887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,7,0.013562666873137156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,31,0.014282666146755219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,31,0.019973333925008774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,63,0.019920000185569126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,63,0.014655999839305878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,127,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,127,0.019850666324297588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,255,0.02266666789849599
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,255,0.023290666441122692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,511,0.03453333427508672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,511,0.033733333150545754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,1023,0.05423999826113383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,2047,0.10798399647076924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,1023,0.0592853327592214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,2047,0.09428800145785014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,4095,0.17559999227523804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,8191,0.4010293483734131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,4095,0.20562666654586792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,8191,0.33804798126220703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,16383,0.9719680150349935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,16383,0.6630239884058634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,float16,32767,3.178154627482096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,1,0.013562666873137156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,3,0.013717333475748697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,1,128,1,float16,fp8,32767,1.4982345898946126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,7,0.013536000003417334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,7,0.015125333021084467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,15,0.014064000298579534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,15,0.016202667107184727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,31,0.014202666779359182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,31,0.020010666300853092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,63,0.019952000429232914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,63,0.014549333602190018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,127,0.016634666671355564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,127,0.019978666057189304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,255,0.022821334501107533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,255,0.023365333676338196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,511,0.01545599972208341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,511,0.012533333152532578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,1023,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,1023,0.014064000298579534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,2047,0.0245919997493426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,2047,0.02565866708755493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,4095,0.02842666705449422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,8191,0.04725333551565806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,8191,0.04450666904449463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,4095,0.030576000610987347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,16383,0.06723733246326447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,16383,0.07347733279069264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,1,0.013552000125249227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,fp8,32767,0.12964800000190735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,1,0.015077333897352219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,2,128,1,float16,float16,32767,0.10644800464312236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,3,0.01360000049074491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,3,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,7,0.013616000612576803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,7,0.015157333264748255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,15,0.014106666048367819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,15,0.016480000068744022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,31,0.014597332725922266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,31,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,63,0.014495999862750372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,63,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,127,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,127,0.0199946661790212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,255,0.012928000340859095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,511,0.013482666263977686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,511,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,1023,0.01945066700379054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,1023,0.012144000579913458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,2047,0.0288426677385966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,2047,0.019914666811625164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,4095,0.04287466903527578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,4095,0.024090667565663654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,8191,0.05990933378537496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,8191,0.04152533411979675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,16383,0.09278399745623271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,16383,0.06354666749636333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,float16,32767,0.13241599996884665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,4,128,1,float16,fp8,32767,0.09896000226338704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,1,0.010842667271693548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,1,0.00620266670982043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,3,0.01179733375708262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,3,0.006570666407545407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,7,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,7,0.006384000182151794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,15,0.011637333780527115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,31,0.010581333190202713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,15,0.007370666911204656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,31,0.008746666833758354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,63,0.01073066641887029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,63,0.006506666541099548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,127,0.010485333700974783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,127,0.008432000254591307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,255,0.013647999614477158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,511,0.019482667247454327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,255,0.009002666920423508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,511,0.009349333122372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,1023,0.02802666773398717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,1023,0.015184000134468079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,2047,0.04321600000063578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,2047,0.018901333212852478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,4095,0.06797333558400472
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,8191,0.08693866928418477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,8191,0.04444266855716705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,4095,0.03019733230272929
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,16383,0.12760532895723978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,16383,0.06821866830190022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,float16,32767,0.21397866805394491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,64,8,128,1,float16,fp8,32767,0.11546132961908977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,1,0.8793333371480306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,1,0.6852533022562662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,3,0.7060266335805258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,3,0.8996000289916992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,7,0.7370399634043375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,7,0.9158720175425211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,15,1.077455997467041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,15,0.9421439965566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,31,1.1045706272125244
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,31,0.9422079722086588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,63,0.950597365697225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,63,1.1162666479746501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,fp8,127,1.133514642715454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,1,128,1,float16,float16,127,1.3350186347961426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,1,0.12232533097267151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,1,0.16983999808629355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,3,0.12243733803431193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,3,0.16991466283798218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,7,0.16994667053222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,7,0.12230400244394939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,15,0.16922666629155478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,31,0.12290666500727336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,15,0.12205866972605388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,63,0.12200533350308736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,31,0.1694773236910502
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,63,0.16990399360656738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,float16,127,0.12257066369056702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,2,128,1,float16,fp8,127,0.18396266301472983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,1,0.16886399189631143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,3,0.1684373418490092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,3,0.11336533228556316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,1,0.1135093371073405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,7,0.1691360076268514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,7,0.11403200030326843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,15,0.16918933391571045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,15,0.11344533165295918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,31,0.16953066984812418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,31,0.11343999703725179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,63,0.11302399635314941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,63,0.16877333323160806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,fp8,127,0.12968533237775168
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,4,128,1,float16,float16,127,0.1695786714553833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,1,0.309989333152771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,3,0.31059734026590985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,1,0.10765332976977031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,3,0.10784533619880676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,7,0.3112906614939372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,15,0.3126239975293477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,7,0.10761599739392598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,15,0.1076800028483073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,31,0.31353066364924115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,31,0.10754133264223735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,63,0.31109867493311566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,float16,127,0.31245867411295575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,63,0.10697600245475769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,64,8,128,1,float16,fp8,127,0.12723732988039652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,1,1.8051573435465496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,1,1.3951733907063801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,3,1.8404587109883626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,7,1.4922560056050618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,7,1.8610186576843262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,3,1.432421366373698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,15,2.160927931467692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,15,1.8855627377827961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,31,2.2017226219177246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,31,1.8826026916503906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,float16,63,2.2253440221150718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,1,128,1,float16,fp8,63,1.8962186177571614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,1,0.3323253393173218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,3,0.23308267196019491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,1,0.23279466231664023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,3,0.332533339659373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,7,0.23228265841801962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,7,0.33313600222269696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,15,0.23236799240112305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,15,0.3323306639989217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,31,0.2323360045750936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,float16,63,0.23090134064356485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,63,0.33290133873621625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,2,128,1,float16,fp8,31,0.33369600772857666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,1,0.32970666885375977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,1,0.21735467513402304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,3,0.3302026589711507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,3,0.21735999981562296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,7,0.3291093309720357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,7,0.21724265813827515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,15,0.21794666846593222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,31,0.32892799377441406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,31,0.21709332863489786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,15,0.32902934153874713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,float16,63,0.32712000608444214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,4,128,1,float16,fp8,63,0.21654399236043295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,1,0.6145653327306112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,1,0.20732265710830688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,3,0.6151626507441202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,3,0.20696000258127847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,15,0.6171146631240845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,7,0.20775999625523886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,15,0.2076639930407206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,31,0.6164373159408569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,7,0.6151093244552612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,1,0.01878400022784869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,1,0.014650666465361914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,3,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,31,0.20787733793258667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,3,0.015146666516860327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,float16,63,0.6128266652425131
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,64,8,128,1,float16,fp8,63,0.20658133427302042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,7,0.01939733326435089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,7,0.015402667224407196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,15,0.021194666624069214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,15,0.019541333119074505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,31,0.01951466624935468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,31,0.021509334444999695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,63,0.021738665799299877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,63,0.019610666980346043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,127,0.02292266736427943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,127,0.02526933451493581
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,255,0.036389333506425224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,1023,0.09593600034713745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,255,0.03311466674009959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,511,0.05406400064627329
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,1023,0.10195199648539226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,511,0.058304001887639366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,2047,0.18900267283121744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,4095,0.36320531368255615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,2047,0.17963733275731406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,4095,0.3457119862238566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,8191,0.7909386952718099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,8191,0.6780052979787191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,fp8,16383,1.4764800071716309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,1,0.01893866683046023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,1,128,1,float16,float16,16383,1.8720107078552246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,1,0.014746667196353277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,15,0.02139200021823247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,3,0.015311999867359797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,7,0.01933866615096728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,3,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,7,0.015482666591803232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,31,0.019445333629846573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,15,0.01951466624935468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,31,0.021525333325068157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,63,0.021829334398110706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,63,0.019551999866962433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,127,0.02535466601451238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,255,0.014682666709025701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,127,0.022869333624839783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,511,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,255,0.013173333058754602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,511,0.012645332763592402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,1023,0.0230880007147789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,2047,0.034661332766215004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,1023,0.02550933261712392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,2047,0.03355200091997782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,4095,0.052373334765434265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,16383,0.11188800136248271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,4095,0.05323733389377594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,float16,8191,0.07379200061162312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,8191,0.09359467029571533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,2,128,1,float16,fp8,16383,0.13588266571362814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,1,0.010640000303586325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,3,0.01073066641887029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,1,0.007146666447321574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,7,0.0106133334338665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,7,0.007280000175038974
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,3,0.008736000085870424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,31,0.010501333822806677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,15,0.010666667173306147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,15,0.007407999907930692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,31,0.007125333572427432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,63,0.010602666685978571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,63,0.007477333148320516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,127,0.008245333408315977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,127,0.011530666301647821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,255,0.013925333817799887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,255,0.009488000224033991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,1023,0.01985599969824155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,511,0.019776000330845516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,1023,0.028736000259717304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,2047,0.04438399771849314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,2047,0.02779199928045273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,4095,0.04669333497683207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,4095,0.06904000043869019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,8191,0.08786666393280029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,8191,0.06128533184528351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,float16,16383,0.12974933783213297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,1,0.008469333251317343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,1,0.010805333654085795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,3,0.011018666128317514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,4,128,1,float16,fp8,16383,0.08920533458391826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,3,0.006437333300709724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,7,0.010773333410422007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,7,0.006501333167155583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,15,0.011813333878914515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,15,0.00660800002515316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,31,0.010928000013033548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,31,0.008645333349704742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,63,0.010944000134865442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,63,0.006480000292261441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,127,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,127,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,255,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,255,0.009248000259200731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,511,0.028186666468779247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,511,0.014965333044528961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,2047,0.06540266672770183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,1023,0.04645866652329763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,1023,0.021925332645575207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,2047,0.028560000161329906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,4095,0.08538132905960083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,4095,0.04303466777006785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,8191,0.1253493328889211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,8191,0.06678933401902516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,fp8,16383,0.11312533418337505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,64,8,128,1,float16,float16,16383,0.21266132593154907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,1,0.018863999595244724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,1,0.014858666807413101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,7,0.01945066700379054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,3,0.01918399954835574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,3,0.01525866612792015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,7,0.015487999965747198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,31,0.021568000316619873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,15,0.01952533299724261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,15,0.02146133283774058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,31,0.019317333896954853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,63,0.021759999295075733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,63,0.01971199984351794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,127,0.02534399926662445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,127,0.022997332115968067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,255,0.011952000359694162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,255,0.014901333798964819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,511,0.015376000354687372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,511,0.012746666868527731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,2047,0.034559999903043113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,1023,0.025434667865435284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,1023,0.022944000860055287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,2047,0.03369066615899404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,4095,0.053157334526379905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,4095,0.053599998354911804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,8191,0.07594133416811626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,8191,0.0940106709798177
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,1,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,float16,16383,0.11460266510645549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,1,128,1,float16,fp8,16383,0.13642133275667825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,1,0.008005333443482717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,3,0.011653333902359009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,3,0.0120319997270902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,7,0.010581333190202713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,7,0.006954666847983996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,15,0.007018666714429855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,15,0.010602666685978571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,31,0.010794666906197866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,31,0.00850133349498113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,63,0.011786667009194693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,63,0.007365333537260692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,127,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,127,0.007514666765928268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,255,0.013669333110253016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,255,0.009429333110650381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,511,0.01953599974513054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,511,0.0122079998254776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,1023,0.028357334434986115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,1023,0.019978666057189304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,2047,0.04539733131726583
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,2047,0.02775466690460841
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,4095,0.0690719981988271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,8191,0.0897653301556905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,4095,0.0462773342927297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,8191,0.06154666841030121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,1,0.010778666784365972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,float16,16383,0.1291039983431498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,2,128,1,float16,fp8,16383,0.08853866656621297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,1,0.0064853330453236895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,3,0.011893333246310553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,7,0.01099733387430509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,3,0.008181333541870117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,7,0.006597333277265231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,15,0.011029332876205444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,15,0.006405333057045937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,31,0.008645333349704742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,63,0.008623999853928884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,127,0.010837333897749582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,127,0.007418666655818622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,255,0.02111999938885371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,255,0.009322666873534521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,511,0.027999999622503918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,511,0.015253332753976187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,1023,0.04570133487383524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,1023,0.022074667116006214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,2047,0.06660266717274983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,2047,0.029781334102153778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,4095,0.0432586669921875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,4095,0.0862506628036499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,8191,0.1265546679496765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,8191,0.06675733129183452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,float16,16383,0.21226666371027628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,4,128,1,float16,fp8,16383,0.11332266529401143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,3,0.006624000146985054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,7,0.006762666627764702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,1,0.00726400005320708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,15,0.006570666407545407
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,63,0.008538666491707167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,1,0.01562133307258288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,3,0.015765332927306492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,63,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,15,0.01579733317097028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,7,0.015552000453074774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,255,0.030165334542592365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,31,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,255,0.013797332843144735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,511,0.04650666813055674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,1023,0.06205333272616068
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,1023,0.030896000564098358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,2047,0.08543466528256734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,127,0.015599999576807022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,127,0.007407999907930692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,511,0.024106666445732117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,4095,0.12667733430862427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,4095,0.0678719977537791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,8191,0.2116640011469523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,8191,0.11392000317573547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,float16,16383,0.3806613286336263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,2047,0.044810667634010315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,32,8,128,1,float16,fp8,16383,0.20518400271733603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,1,0.008341333518425623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,1,0.009381333366036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,3,0.0106133334338665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,3,0.008389333263039589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,7,0.008389333263039589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,7,0.009445333232482275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,15,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,31,0.011258666714032492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,31,0.009306666751702627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,63,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,15,0.009445333232482275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,127,0.012047999848922094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,127,0.009370666618148485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,63,0.012096000214417776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,255,0.01156266654531161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,511,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,255,0.01211200033624967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,511,0.016480000068744022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,1023,0.017808000246683758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,1023,0.017653333644072216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,2047,0.029648000995318096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,2047,0.029440000653266907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,4095,0.040378667414188385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,4095,0.039701332648595176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,8191,0.022426667312781017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,8191,0.022154666483402252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,16383,0.024725332856178284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,16383,0.026869334280490875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,32767,0.0303413321574529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,32767,0.03408533334732056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,1,0.008336000144481659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,float16,65535,0.03750933210055033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,1,128,1,float16,fp8,65535,0.04366933306058248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,1,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,7,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,3,0.008517333616813024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,7,0.008367999767263731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,15,0.008645333349704742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,31,0.010362666721145311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,15,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,31,0.009381333366036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,63,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,63,0.012058666596810022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,255,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,127,0.010485333700974783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,511,0.016597333053747814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,255,0.012175999581813812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,1023,0.01732800031701724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,511,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,1023,0.01783466711640358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,2047,0.029631999631722767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,8191,0.01937599976857503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,4095,0.015626666446526844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,2047,0.02958400050799052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,4095,0.013386666774749756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,16383,0.020687999824682873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,8191,0.015935999651749928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,16383,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,32767,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,32767,0.024480000138282776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,fp8,65535,0.030965333183606465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,1,0.008186666915814081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,2,128,1,float16,float16,65535,0.0315733328461647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,1,0.009328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,3,0.008367999767263731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,3,0.00938666673998038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,7,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,15,0.008592000231146812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,31,0.009328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,63,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,7,0.009530666594703993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,31,0.010255999863147736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,255,0.010431999961535135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,127,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,127,0.012069333344697952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,255,0.0122079998254776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,511,0.01655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,511,0.016650666793187458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,1023,0.017701332767804463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,1023,0.017711999515692394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,2047,0.015103999525308609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,2047,0.009626666704813639
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,8191,0.012938667088747025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,4095,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,4095,0.016058667252461117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,8191,0.019354666272799175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,16383,0.02180800090233485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,32767,0.021514666577180225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,16383,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,32767,0.0349386657277743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,float16,65535,0.04965866605440775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,1,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,4,128,1,float16,fp8,65535,0.02716800073782603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,1,0.009445333232482275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,3,0.008256000156203905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,3,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,7,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,31,0.010191999996701876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,7,0.009813333551088968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,15,0.0086666668454806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,31,0.00922133338948091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,63,0.009317333499590555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,127,0.009258666386206945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,63,0.012335999558369318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,255,0.010474666953086853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,255,0.012272000312805176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,511,0.016565332810084026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,511,0.016458666572968166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,15,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,1023,0.008170666793982187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,4095,0.01062400018175443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,1023,0.013722666849692663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,2047,0.009343999748428663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,2047,0.015226667126019796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,4095,0.016480000068744022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,8191,0.025477332373460133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,16383,0.03120533376932144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,8191,0.014298666268587112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,16383,0.016970666746298473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,32767,0.05087999999523163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,32767,0.03268266717592875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,float16,65535,0.07745600243409474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,1,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,32,8,128,1,float16,fp8,65535,0.04428799947102865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,1,0.009375999992092451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,3,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,3,0.008389333263039589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,7,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,7,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,15,0.008810666700204214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,15,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,31,0.01032533310353756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,31,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,63,0.012154666086037954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,127,0.012165332833925882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,63,0.009375999992092451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,127,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,1023,0.029637334247430164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,255,0.012138667205969492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,255,0.010618666807810465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,511,0.016607999801635742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,511,0.016901332885026932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,2047,0.03751466671625773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,1023,0.029477333029111225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,2047,0.03697066754102707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,4095,0.019093333433071773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,4095,0.01838933303952217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,8191,0.023157333334287006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,8191,0.022005334496498108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,16383,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,16383,0.026837334036827087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,32767,0.031514666974544525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,32767,0.034346667428811394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,float16,65535,0.04120533416668574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,1,0.008559999987483025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,3,0.009370666618148485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,1,128,1,float16,fp8,65535,0.043978666265805565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,3,0.00847999999920527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,1,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,7,0.008767999708652496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,15,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,15,0.008799999952316284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,7,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,31,0.009290666629870733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,63,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,31,0.010437333335479101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,255,0.010640000303586325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,511,0.016832000265518825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,1023,0.029669334491093952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,511,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,1023,0.029887999097506206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,2047,0.011472000430027643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,8191,0.0205226664741834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,2047,0.01562133307258288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,4095,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,4095,0.01332266628742218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,16383,0.022463999688625336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,8191,0.01609066625436147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,32767,0.03739733248949051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,16383,0.01933866615096728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,32767,0.028815999627113342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,1,0.008837333569924036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,float16,65535,0.0521066685517629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,2,128,1,float16,fp8,65535,0.03573333223660787
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,1,0.009775999933481216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,3,0.008463999877373377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,7,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,3,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,7,0.008367999767263731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,15,0.008549333239595095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,15,0.009882666791478792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,31,0.009450666606426239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,63,0.009450666606426239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,31,0.01033599985142549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,63,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,127,0.009488000224033991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,255,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,127,0.012133333832025528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,511,0.01682666689157486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,255,0.01227733368674914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,511,0.016634666671355564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,1023,0.013701333353916803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,4095,0.017018667111794155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,1023,0.008709333216150602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,8191,0.02629333237806956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,2047,0.01597333326935768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,2047,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,4095,0.010981333752473196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,16383,0.03159466634194056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,8191,0.01470400020480156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,32767,0.051701332132021584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,16383,0.017279999951521557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,32767,0.028058665494124096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,float16,65535,0.07960000137488048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,4,128,1,float16,fp8,65535,0.05343466500441233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,1,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,3,0.008570666735370954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,1,0.009530666594703993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,3,0.00984533317387104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,7,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,7,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,31,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,15,0.008618666479984919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,31,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,63,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,255,0.01080000028014183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,63,0.012522666404644648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,127,0.009466666728258133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,127,0.012346666306257248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,255,0.012319999436537424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,511,0.01293333371480306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,1023,0.013354666531085968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,511,0.007674666742483775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,1023,0.008549333239595095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,2047,0.020554666717847187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,2047,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,4095,0.024245334168275196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,8191,0.03537066777547201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,4095,0.011882666498422623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,8191,0.02081599955757459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,16383,0.05347733199596405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,32767,0.0822026679913203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,16383,0.026127999027570088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,32767,0.04710933566093445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,float16,65535,0.13487466176350912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,1,0.012341332932313284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,32,8,128,1,float16,fp8,65535,0.07555200159549713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,1,0.00915733352303505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,7,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,3,0.012272000312805176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,3,0.010015999898314476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,7,0.012634667257467905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,15,0.012410666793584824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,15,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,31,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,31,0.012309333930412928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,63,0.012202666451533636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,63,0.009829333052039146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,127,0.012458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,127,0.009946666657924652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,255,0.012133333832025528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,255,0.016309333344300587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,511,0.023631999890009563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,511,0.02271466702222824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,1023,0.03389866650104523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,1023,0.041637333730856575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,2047,0.05914133290449778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,2047,0.05563200016816457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,4095,0.08584533135096233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,4095,0.11241599917411804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,float16,8191,0.11104533076286316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,1,128,1,float16,fp8,8191,0.13774399956067404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,1,0.01139733319481214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,1,0.007882666463653246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,3,0.011274666835864386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,7,0.007936000203092894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,7,0.011061333119869232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,3,0.007690666864315669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,31,0.011071999867757162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,15,0.011183999478816986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,15,0.007167999943097432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,31,0.007823999971151352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,63,0.010970667004585266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,127,0.008661333471536636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,63,0.007237333183487256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,127,0.011242666592200598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,255,0.021541332205136616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,511,0.028768000503381092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,511,0.018810667097568512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,1023,0.047744000951449074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,1023,0.032960000137488045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,2047,0.0676800012588501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,2047,0.045498669147491455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,4095,0.08729599912961324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,float16,8191,0.12897599736849466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,8191,0.08782399694124858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,2,128,1,float16,fp8,4095,0.06057600180308024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,1,0.01599466676513354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,3,0.015706667055686314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,1,0.007135999699433644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,3,0.006805333619316419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,7,0.01588800052801768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,7,0.007045333584149678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,15,0.01598400001724561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,15,0.008143999924262365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,31,0.015893333901961643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,31,0.007226666435599327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,63,0.007365333537260692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,127,0.01589866727590561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,63,0.01595199977358182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,127,0.007584000006318092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,255,0.03052799900372823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,255,0.015029333531856537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,511,0.047653332352638245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,511,0.02170666555563609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,1023,0.061994666854540505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,1023,0.028421332438786823
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,2047,0.08549867073694865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,2047,0.04324266811211904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,4095,0.1272213359673818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,8191,0.11326932907104492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,float16,8191,0.21424533923467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,4,128,1,float16,fp8,4095,0.0666186660528183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,1,0.025455998877684276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,1,0.009786666681369146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,3,0.025349333882331848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,7,0.025386666258176167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,3,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,7,0.009877333417534828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,15,0.025829332570234936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,63,0.025114665428797405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,31,0.025610665480295818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,31,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,63,0.009621333330869675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,127,0.025685332715511322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,255,0.02390933285156886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,511,0.06473599870999654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,511,0.03176533430814743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,255,0.050288001696268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,1023,0.08241066833337148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,1023,0.045408000548680626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,2047,0.06811200082302094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,2047,0.12892799576123556
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,4095,0.21626132726669312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,4095,0.11441600322723389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,float16,8191,0.38493335247039795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,1,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,32,8,128,1,float16,fp8,8191,0.20574933290481567
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,1,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,3,0.008592000231146812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,15,0.008698666468262672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,3,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,7,0.008645333349704742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,7,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,15,0.009904000287254652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,31,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,63,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,31,0.010474666953086853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,127,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,63,0.012213333199421564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,127,0.012378666549921036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,255,0.010666667173306147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,255,0.01232533281048139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,511,0.02807466685771942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,511,0.02805333336194356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,2047,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,1023,0.03277866790692011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,1023,0.03265066693226496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,2047,0.015642666568358738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,4095,0.018197332819302876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,4095,0.020655999581019085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,8191,0.02518400053183238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,8191,0.022431999444961548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,32767,0.048709332942962646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,16383,0.027461332579453785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,16383,0.026911998788515728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,32767,0.04970666766166687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,float16,65535,0.06596800188223521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,1,0.008623999853928884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,7,0.008703999842206636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,7,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,3,0.008527999743819237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,3,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,1,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,1,128,1,float16,fp8,65535,0.06822933256626129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,31,0.009392000113924345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,15,0.009786666681369146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,31,0.010464000205198923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,15,0.008976000050703684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,63,0.012304000556468964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,63,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,127,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,127,0.012245333443085352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,255,0.010672000547250112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,511,0.02809600035349528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,255,0.012367999802033106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,511,0.02794133375088374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,1023,0.010298666854699453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,1023,0.013770667215188345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,2047,0.01740266631046931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,2047,0.011717333147923151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,4095,0.018800000349680584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,4095,0.013418667018413544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,8191,0.027893332143624622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,8191,0.01868266612291336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,16383,0.02298133323589961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,16383,0.03408000121514002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,32767,0.055200000603993736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,32767,0.042090664307276406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,float16,65535,0.0835040012995402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,1,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,3,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,1,0.0084906667470932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,2,128,1,float16,fp8,65535,0.05712000032265981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,3,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,15,0.009775999933481216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,7,0.0085333331177632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,7,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,15,0.008826666822036108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,31,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,31,0.010501333822806677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,63,0.009701333319147428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,63,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,255,0.010645333677530289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,127,0.012517333030700684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,127,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,511,0.012965332716703415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,255,0.012362666428089142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,511,0.0081386665503184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,1023,0.01368533323208491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,1023,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,2047,0.010858666151762009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,2047,0.0222080002228419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,4095,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,4095,0.012794667234023413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,8191,0.01782400036851565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,16383,0.05538133283456167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,8191,0.0365280012289683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,16383,0.029520000020662945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,32767,0.08332799871762593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,1,0.008432000254591307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,32767,0.04450666904449463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,fp8,65535,0.07499733567237854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,4,128,1,float16,float16,65535,0.13758400082588196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,1,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,3,0.009663999701539675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,3,0.008736000085870424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,7,0.0086666668454806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,15,0.008746666833758354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,7,0.009813333551088968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,15,0.00985599992175897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,31,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,31,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,127,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,63,0.012426666915416718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,63,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,127,0.01232533281048139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,255,0.012709333250919977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,255,0.007311999797821045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,511,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,1023,0.01934933289885521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,511,0.013359999905029932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,1023,0.009381333366036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,2047,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,2047,0.028666667640209198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,4095,0.01798933371901512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,4095,0.041434665520985924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,8191,0.06007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,8191,0.02886933336655299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,16383,0.08925333619117737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,16383,0.048309331138928734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,32767,0.12934933106104532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,32767,0.07796266674995422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,float16,65535,0.2209333380063375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,32,8,128,1,float16,fp8,65535,0.13176533579826355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,1,0.013125333935022354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,1,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,3,0.013199999928474426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,3,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,7,0.013157332936922709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,15,0.013072000195582708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,7,0.009354666496316591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,15,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,31,0.013082666943470636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,127,0.012965332716703415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,31,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,63,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,63,0.013045333325862885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,127,0.010277333358923594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,255,0.025973332424958546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,511,0.03496533383925756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,255,0.02125866711139679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,1023,0.05825066566467285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,511,0.04005866746107737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,1023,0.07398933172225952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,2047,0.08450667063395183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,2047,0.11154133081436157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,float16,4095,0.11149332920710246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,1,0.016501333564519882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,1,128,1,float16,fp8,4095,0.13878400127092996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,1,0.00784533346692721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,3,0.016538667182127636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,3,0.007850666840871176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,15,0.01646399994691213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,7,0.016442666451136272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,7,0.007765333478649457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,15,0.007871999715765318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,31,0.01666133354107539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,31,0.007882666463653246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,63,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,127,0.008778666456540426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,63,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,127,0.01646399994691213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,255,0.03156800071398417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,255,0.018031999468803406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,511,0.03233599911133448
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,511,0.048911998669306435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,1023,0.06362666686375935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,1023,0.04534400006135305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,2047,0.08741333087285359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,2047,0.06048533320426941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,fp8,4095,0.08787199854850769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,1,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,1,0.02603733291228612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,2,128,1,float16,float16,4095,0.1307413379351298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,3,0.026170666019121807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,3,0.010485333700974783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,15,0.025968000292778015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,7,0.02613866577545802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,7,0.010464000205198923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,15,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,31,0.026421333352724712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,31,0.010480000327030817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,63,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,63,0.025941332181294758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,127,0.01231466606259346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,127,0.026362667481104534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,255,0.05073066552480062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,255,0.021957332889238994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,511,0.06545066833496094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,511,0.030213333666324615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,1023,0.043493335445721946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,1023,0.08434133728345235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,2047,0.13089066743850708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,2047,0.0670666644970576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,1,0.04470933477083842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,float16,4095,0.21731199820836386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,1,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,4,128,1,float16,fp8,4095,0.11429333686828613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,3,0.045082668463389076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,3,0.016586666305859882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,15,0.016575999557971954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,7,0.044922664761543274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,7,0.01658133293191592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,15,0.04524266719818115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,31,0.04478933413823446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,31,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,63,0.04460266729195913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,63,0.016480000068744022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,127,0.019354666272799175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,127,0.04560533165931702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,255,0.053727999329566956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,255,0.026885333160559338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,1023,0.11641599734624226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,511,0.07361066838105519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,511,0.040261333187421165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,1023,0.06362666686375935
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,2047,0.2058186729749044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,2047,0.10988799730936687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,3,0.02040533348917961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,1,0.020319999506076176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,float16,4095,0.37491198380788165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,1,0.015989333391189575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,32,8,128,1,float16,fp8,4095,0.20086399714152017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,3,0.016234666109085083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,7,0.02094399929046631
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,7,0.016063999384641647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,15,0.020319999506076176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,31,0.02045866722861926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,15,0.0161920003592968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,31,0.01616000011563301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,63,0.020021333048741024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,63,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,127,0.017792000124851864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,127,0.020245333512624104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,255,0.03896533449490865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,511,0.06052800019582113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,255,0.038986665507157646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,float16,1023,0.07914133369922638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,511,0.08038933575153351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,1,128,1,float16,fp8,1023,0.11106666922569275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,1,0.012938667088747025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,1,0.027221334477265675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,3,0.02717866748571396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,3,0.012906666845083237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,7,0.02784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,15,0.027280000348885853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,15,0.012896000097195307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,7,0.012906666845083237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,31,0.012965332716703415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,31,0.027119999130566914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,63,0.027248000105222065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,63,0.012938667088747025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,127,0.027263998985290527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,127,0.014938666174809137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,255,0.03175999969244003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,255,0.05279466509819031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,511,0.06770666440327962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,511,0.0469760000705719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,float16,1023,0.08648533622423808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,1,0.04625066618124644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,2,128,1,float16,fp8,1023,0.061247999469439186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,15,0.046181331078211464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,7,0.046309332052866616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,3,0.046021332343419395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,7,0.015178666760524115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,15,0.015173333386580149
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,31,0.04659200211366018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,31,0.014981333166360855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,63,0.014991999914248785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,63,0.04573333263397217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,127,0.017968000223239262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,127,0.047557334105173744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,255,0.05578133463859558
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,255,0.026687999566396076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,511,0.07493866483370464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,511,0.04074666649103165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,fp8,1023,0.06364800035953522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,1,0.03010133405526479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,3,0.030282666285832722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,4,128,1,float16,float16,1023,0.11841600139935811
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,3,0.08172266681989034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,1,0.08182399968306224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,15,0.030042665700117748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,7,0.08210666477680206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,7,0.030074665943781536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,15,0.08150400221347809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,31,0.030058667063713074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,31,0.08160533507664998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,63,0.08179200192292531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,63,0.030117332935333252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,127,0.08335999647776286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,127,0.038912000755469
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,255,0.09545600414276123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,255,0.052245333790779114
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,1023,0.21254400412241617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,float16,511,0.13090133666992188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,511,0.07525866727034251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,32,8,128,1,float16,fp8,1023,0.12133866548538208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,1,0.035616000493367515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,1,0.02939733366171519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,3,0.029338667790095013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,3,0.034927998979886375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,7,0.03495466709136963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,31,0.029215998947620392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,15,0.03498133271932602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,31,0.03498133271932602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,15,0.02937600016593933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,7,0.02926933268706004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,63,0.03484266748030981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,127,0.035114665826161705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,63,0.02918400118748347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,255,0.06724800169467926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,127,0.03264000018437704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,255,0.0865760048230489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,float16,511,0.08416000008583069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,1,0.048058668772379555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,1,128,1,float16,fp8,511,0.11639466881752014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,3,0.048298666874567665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,1,0.022800001005331676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,15,0.048138668139775596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,7,0.04861866434415182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,3,0.022805333137512207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,7,0.022826666633288067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,15,0.02292266736427943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,63,0.04789866507053375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,31,0.04828799764315287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,31,0.022618666291236877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,63,0.022778667509555817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,255,0.059104000528653465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,127,0.027509334186712902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,127,0.049584001302719116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,255,0.04119466741879781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,float16,511,0.0783786674340566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,2,128,1,float16,fp8,511,0.056373332937558494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,1,0.08408000071843465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,1,0.026047999660174053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,7,0.08400533596674602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,3,0.08378666639328003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,15,0.08368000388145447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,3,0.02664000044266383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,7,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,15,0.02640533447265625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,31,0.084389328956604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,31,0.026501332720120747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,63,0.026389333109060924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,63,0.08488532900810242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,255,0.09893332918485005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,127,0.037248000502586365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,127,0.08629866441090901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,511,0.07047466437021892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,fp8,255,0.048997332652409874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,4,128,1,float16,float16,511,0.1355893313884735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,1,0.06505600114663442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,1,0.15169066190719604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,7,0.15147200226783752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,3,0.15130133430163065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,3,0.06572266419728597
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,15,0.1516480048497518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,7,0.06550399959087372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,15,0.06484266618887584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,31,0.06530133386452992
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,127,0.15450132886568704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,31,0.15465600291887918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,63,0.1548960010210673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,63,0.06538666784763336
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,255,0.09760533769925435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,255,0.17813867330551147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,127,0.07576000193754832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,float16,511,0.24577067295710245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,1,0.013989333063364029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,32,8,128,1,float16,fp8,511,0.1432906687259674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,1,0.010122666756312052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,3,0.013983999689420065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,3,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,15,0.01441066712141037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,7,0.009226666763424873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,15,0.00978133330742518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,7,0.013951999445756277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,31,0.01179733375708262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,63,0.011706666400035223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,31,0.015834666788578033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,63,0.016037333756685257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,127,0.016261332978804905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,127,0.011823999385039011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,511,0.02475200096766154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,255,0.018288000176350277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,1023,0.01392000044385592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,1023,0.01565333331624667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,2047,0.019760000209013622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,511,0.018885333091020584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,4095,0.021909333765506744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,2047,0.015856000284353893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,4095,0.01830400029818217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,16383,0.0402399996916453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,8191,0.033189333975315094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,8191,0.032746667663256325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,32767,0.06775466601053874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,32767,0.07257066667079926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,16383,0.04015466570854187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,1,0.014015999933083853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,1,0.008986666798591614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,3,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,fp8,65535,0.11210667093594869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,3,0.014080000420411428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,1,128,1,float16,float16,65535,0.10086933771769206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,7,0.014042666802803675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,15,0.009941333283980688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,31,0.015850666910409927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,7,0.00902399979531765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,15,0.0144213338692983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,127,0.016058667252461117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,63,0.01607999950647354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,63,0.011786667009194693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,255,0.018325333793958027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,127,0.01181866725285848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,255,0.013482666263977686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,511,0.013093333691358566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,511,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,2047,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,2047,0.013232000172138214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,1023,0.013621332744757334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,1023,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,4095,0.02571733295917511
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,4095,0.015669333438078564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,8191,0.038015998899936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,8191,0.027386667827765148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,16383,0.05676266551017761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,16383,0.0352960005402565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,32767,0.06090133388837179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,32767,0.08433600266774495
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,float16,65535,0.1383519967397054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,2,128,1,float16,fp8,65535,0.09647466739018758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,1,0.014042666802803675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,1,0.008954666554927826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,15,0.009904000287254652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,3,0.014058666924635569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,3,0.009130666653315226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,15,0.014416000495354334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,7,0.014127999544143677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,7,0.009103999783595404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,31,0.015674666812022526
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,31,0.011765333513418833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,127,0.016149333367745083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,63,0.011850666254758835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,255,0.012565333396196365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,127,0.011999999483426413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,63,0.016069332758585613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,255,0.007781333600481351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,511,0.013199999928474426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,511,0.008421333506703377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,1023,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,1023,0.019093333433071773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,4095,0.01966933285196622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,4095,0.042021334171295166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,2047,0.02880000074704488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,2047,0.01268799975514412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,8191,0.059018666545550026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,8191,0.026165333886941273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,16383,0.04649066428343455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,32767,0.13083199659983316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,32767,0.06869866450627644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,16383,0.09101866682370503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,float16,65535,0.22180267175038657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,4,128,1,float16,fp8,65535,0.11409599582354228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,1,0.006064000229040782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,1,0.010773333410422007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,3,0.01051733394463857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,3,0.005957333371043205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,7,0.010634666929642359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,7,0.006085333103934924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,15,0.006090666477878888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,15,0.0106133334338665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,31,0.006058666855096817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,31,0.010634666929642359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,63,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,127,0.006624000146985054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,63,0.006133333469430606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,127,0.010773333410422007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,255,0.013376000026861826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,255,0.00761600024998188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,511,0.019306667149066925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,1023,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,511,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,2047,0.04348266621430715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,2047,0.02089066555102666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,1023,0.015354666858911514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,4095,0.06803733110427856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,4095,0.03313066562016805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,16383,0.06980266670385997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,8191,0.08640000224113464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,8191,0.046869332591692604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,16383,0.12788266936937967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,32767,0.21422400077184042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,32767,0.11666133006413777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,float16,65535,0.3808693488438924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,32,8,128,1,float16,fp8,65535,0.21029865741729736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,1,0.06347733239332835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,1,0.05669866502285004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,3,0.06346666812896729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,7,0.06343466540177663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,3,0.058058664202690125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,7,0.05666666726271311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,15,0.06389333307743073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,15,0.05685333410898844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,31,0.06464533507823944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,63,0.06485866506894429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,63,0.058335999647776283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,31,0.05779199798901876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,127,0.07667733232180278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,127,0.06600533425807953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,1,0.08814932902654012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,1,0.048250665267308555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,fp8,255,0.10752532879511516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,1,128,1,float16,float16,255,0.07682133217652638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,3,0.048453330993652344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,7,0.08820799986521403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,3,0.08803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,7,0.046816001335779824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,31,0.04834666848182678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,15,0.08874133229255676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,15,0.04861866434415182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,31,0.08971200386683147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,63,0.0895253320535024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,63,0.04784533381462097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,127,0.06786666810512543
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,127,0.09052800138791402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,float16,255,0.10567466417948405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,2,128,1,float16,fp8,255,0.08028799792130788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,1,0.057461331288019814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,1,0.15583466490109762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,3,0.057429333527882896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,3,0.15596800049146017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,7,0.15685866276423135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,15,0.15786133209864298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,7,0.05771199862162272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,15,0.057573333382606506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,31,0.05765866736570994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,63,0.05745600163936615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,31,0.1604159971078237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,127,0.16140799721082053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,63,0.1609280010064443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,127,0.06758933266003926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,fp8,255,0.08749333024024963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,4,128,1,float16,float16,255,0.1868213415145874
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,1,0.2937013308207194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,3,0.29526400566101074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,3,0.12551466623942056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,7,0.29682666063308716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,1,0.12552000085512796
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,7,0.1258080005645752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,31,0.3030719955762227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,15,0.30107200145721436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,15,0.12574932972590128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,31,0.12564266721407572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,63,0.12600533167521158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,63,0.301258663336436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,127,0.2980533242225647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,127,0.1432960033416748
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,fp8,255,0.18760534127553305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,32,8,128,1,float16,float16,255,0.345792015393575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,1,0.12177600463231404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,1,0.16954133907953897
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,3,0.12215466300646464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,3,0.16994667053222656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,7,0.12198399504025777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,15,0.12180266777674358
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,7,0.16979734102884927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,15,0.16980266571044922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,31,0.12251200278600057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,31,0.1700213352839152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,63,0.12223999698956807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,63,0.16937067111333212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,1,0.16710400581359863
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,3,0.16815465688705444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,float16,127,0.12247467041015625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,1,0.11356799801190694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,1,128,1,float16,fp8,127,0.1846239964167277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,3,0.1134986678759257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,7,0.16793066263198853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,7,0.11332799990971883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,15,0.16938666502634683
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,15,0.11329066753387451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,31,0.16972267627716064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,31,0.11351999640464783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,63,0.11301866173744202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,63,0.1681493322054545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,float16,127,0.17037334044774374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,2,128,1,float16,fp8,127,0.12985066572825113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,1,0.310479998588562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,3,0.10745599865913391
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,1,0.10760000348091125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,3,0.3099626700083415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,7,0.10785599549611409
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,7,0.31043734153111774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,15,0.3126346667607625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,15,0.1076586643854777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,31,0.31282132863998413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,63,0.31067200501759845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,63,0.10756267110506694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,31,0.10760000348091125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,fp8,127,0.12727999687194824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,1,0.5931413173675537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,1,0.2439840038617452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,4,128,1,float16,float16,127,0.31224000453948975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,3,0.5921546618143717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,7,0.24462932348251343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,3,0.2446133295694987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,7,0.5953226486841837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,15,0.5984533230463663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,15,0.24387733141581217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,31,0.5968266725540161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,1,0.013424000392357508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,1,0.014752000570297241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,63,0.5918346643447876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,3,0.013829333086808523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,31,0.24426132440567017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,127,0.27933865785598755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,fp8,63,0.24479466676712036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,3,0.01492799942692121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,32,8,128,1,float16,float16,127,0.5820159912109375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,7,0.013728000223636627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,7,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,15,0.016384000579516094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,15,0.0143306665122509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,31,0.01452800010641416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,31,0.019866666446129482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,63,0.020074666788180668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,63,0.014533333480358124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,127,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,127,0.01987733319401741
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,511,0.014981333166360855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,255,0.022597332795461018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,255,0.023247999449570973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,511,0.012741333494583765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,1023,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,1023,0.013989333063364029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,2047,0.02606933315594991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,2047,0.024341332415739696
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,8191,0.04460800190766653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,4095,0.030218665798505146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,4095,0.028490667541821797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,16383,0.06710400183995564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,8191,0.04736533264319102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,16383,0.07727999985218048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,3,0.013770667215188345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,1,0.013455999394257864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,float16,32767,0.10782933235168457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,1,0.01481066644191742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,1,128,1,float16,fp8,32767,0.12935466567675272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,7,0.01370666672786077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,63,0.014671999961137772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,7,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,15,0.014202666779359182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,15,0.01639466608564059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,31,0.014352000008026758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,31,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,127,0.016528000434239704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,63,0.019989332805077236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,127,0.019925333559513092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,255,0.00914666677514712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,255,0.012709333250919977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,511,0.013541333377361298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,511,0.01089599976936976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,1023,0.01940800001223882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,1023,0.011850666254758835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,2047,0.019973333925008774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,2047,0.02902399996916453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,8191,0.04104000081618627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,4095,0.04261333247025808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,4095,0.02418133368094762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,8191,0.06055466830730438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,16383,0.0928106705347697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,16383,0.0634933312733968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,fp8,32767,0.09910933176676433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,2,128,1,float16,float16,32767,0.13185066978136697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,3,0.011567999919255575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,1,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,1,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,7,0.006495999793211619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,7,0.010464000205198923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,3,0.0064853330453236895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,15,0.010677333921194077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,15,0.008512000242869059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,31,0.010698666175206503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,63,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,63,0.0064266665528217954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,31,0.007354666789372762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,127,0.010575999816258749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,127,0.006837333242098491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,255,0.013647999614477158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,511,0.019285333653291065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,255,0.00808533343176047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,511,0.009525333220760027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,1023,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,2047,0.04363200068473816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,4095,0.06742933392524719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,1023,0.027877333263556164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,2047,0.019050666441520054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,4095,0.029296000798543293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,8191,0.0872373382250468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,8191,0.04382933179537455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,16383,0.06746666630109151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,16383,0.12731732924779257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,1,0.010778666784365972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,float16,32767,0.21369065841039023
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,4,128,1,float16,fp8,32767,0.11485866705576579
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,3,0.006117333347598712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,1,0.0061493335912625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,3,0.010735999792814255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,7,0.010735999792814255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,7,0.006250666454434395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,31,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,15,0.00847999999920527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,15,0.010885333021481832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,31,0.006319999694824219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,63,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,63,0.006090666477878888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,127,0.007002666592597961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,127,0.010618666807810465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,511,0.027957332630952198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,255,0.020799999435742695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,255,0.010053333515922228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,1023,0.02455466737349828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,511,0.01443733274936676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,1023,0.04597333570321401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,2047,0.06604266663392384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,8191,0.1255519986152649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,2047,0.03219733387231827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,4095,0.08521067102750142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,4095,0.04532266656557719
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,8191,0.06805333495140076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,16383,0.21094934145609537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,16383,0.11434132854143779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,1,0.01358933374285698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,float16,32767,0.3797706762949626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,32,8,128,1,float16,fp8,32767,0.20535467068354288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,1,0.014954666296641031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,3,0.013898666948080063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,7,0.01393066719174385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,7,0.015125333021084467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,15,0.01423466702302297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,15,0.01632000009218852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,31,0.014314666390419006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,31,0.020010666300853092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,127,0.016549333930015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,63,0.014597332725922266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,127,0.019946667055288952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,63,0.019920000185569126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,255,0.013007999708255133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,255,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,511,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,1023,0.011989332735538483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,511,0.013568000247081121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,1023,0.019365333020687103
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,2047,0.02951466788848241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,2047,0.019941333681344986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,4095,0.04338666796684265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,4095,0.02430933217207591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,8191,0.06141333281993866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,8191,0.04035733391841253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,16383,0.0938933293024699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,16383,0.06418133278687795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,1,0.010645333677530289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,1,0.006469333544373512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,float16,32767,0.13360533118247986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,1,128,1,float16,fp8,32767,0.0983679989973704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,3,0.010575999816258749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,3,0.006362666686375936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,7,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,7,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,15,0.010330666477481524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,15,0.00655466690659523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,31,0.010565333068370819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,31,0.006175999840100606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,127,0.0103946669648091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,63,0.006405333057045937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,63,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,127,0.008261333530147871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,255,0.013557333499193192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,511,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,255,0.008069333309928576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,511,0.019274666905403137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,2047,0.04409066836039225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,1023,0.0277813325325648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,1023,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,4095,0.0682666649421056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,2047,0.018911999960740406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,8191,0.043968002001444496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,4095,0.030261332790056866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,8191,0.08813866972923279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,16383,0.12830400466918945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,16383,0.06819733480612437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,1,0.010714666297038397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,float16,32767,0.2139093279838562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,1,0.006128000095486641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,3,0.010805333654085795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,2,128,1,float16,fp8,32767,0.11486933628718059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,3,0.006362666686375936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,7,0.006239999706546466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,7,0.0107893335322539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,15,0.010703999549150467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,31,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,15,0.008154666672150293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,31,0.006197333335876465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,63,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,63,0.00814933329820633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,127,0.010581333190202713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,127,0.006720000257094701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,1023,0.04563733438650767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,255,0.02081599955757459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,511,0.02794666588306427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,511,0.014373333503802618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,255,0.009066666786869368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,1023,0.024703999360402424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,2047,0.06630399823188782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,2047,0.0317546675602595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,4095,0.08532800277074178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,8191,0.12774933377901712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,8191,0.06868266562620799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,4095,0.045312002301216125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,16383,0.11435199777285258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,16383,0.21061333020528158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,float16,32767,0.3813279867172241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,4,128,1,float16,fp8,32767,0.20573333899180093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,1,0.006800000245372455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,3,0.006965333595871925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,7,0.006688000013430913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,15,0.006757333253820737
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,31,0.0069759997228781385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,63,0.006682666639486949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,1,0.015626666446526844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,7,0.01587733378012975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,127,0.015429332852363586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,31,0.015552000453074774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,15,0.015642666568358738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,255,0.030410667260487873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,255,0.013733333597580591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,63,0.015386667102575302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,511,0.04659733176231384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,127,0.008090666805704435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,3,0.01534933348496755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,1023,0.030479999879995983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,2047,0.04399466514587402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,1023,0.062047998110453285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,2047,0.0853760043780009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,4095,0.12590400377909342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,511,0.023792001108328503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,8191,0.21086400747299194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,8191,0.11296000083287557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,16383,0.20389332373936972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,16383,0.3784799973169963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,1,0.00820266641676426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,float16,32767,0.7153173287709554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,1,0.009434666484594345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,32767,0.3858400185902913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,3,0.008367999767263731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,3,0.009141333401203156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,7,0.008112000301480293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,7,0.009183999771873156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,15,0.008693333094318708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,15,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,63,0.009237333511312803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,31,0.0100426667680343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,31,0.009039999917149544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,63,0.012015999605258306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,16,8,128,1,float16,fp8,4095,0.06717333197593689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,127,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,127,0.01192533348997434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,255,0.010357333347201347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,511,0.01657066618402799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,255,0.01181866725285848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,511,0.01664000004529953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,1023,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,1023,0.017530667285124462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,4095,0.029733332494894665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,2047,0.017456000049908955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,2047,0.017797333498795826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,4095,0.029274667302767437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,8191,0.01863466699918111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,8191,0.016016000260909397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,16383,0.019082666685183842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,16383,0.020442667106787365
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,32767,0.024469333390394848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,65535,0.029338667790095013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,32767,0.02422933280467987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,65535,0.030810666580994923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,1,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,float16,131071,0.04087999959786733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,1,0.008277333031098047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,1,128,1,float16,fp8,131071,0.04083200047413508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,3,0.008181333541870117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,3,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,7,0.008256000156203905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,7,0.009269333134094873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,15,0.00842666688064734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,31,0.009098666409651438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,15,0.009541333342591921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,31,0.010175999874869982
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,63,0.01211200033624967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,63,0.009317333499590555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,127,0.009285333255926767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,255,0.010224000240365664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,127,0.01210133358836174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,255,0.012042666474978128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,511,0.016490666816631954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,2047,0.017914666483799618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,1023,0.017871999492247898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,511,0.016341333587964375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,1023,0.017322666943073273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,2047,0.017423999806245167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,4095,0.015925332903862
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,8191,0.019093333433071773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,8191,0.012896000097195307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,4095,0.01080000028014183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,16383,0.02006400004029274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,16383,0.015184000134468079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,32767,0.02475200096766154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,32767,0.019359999646743137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,65535,0.030933332939942677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,1,0.008154666672150293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,65535,0.024266667664051056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,float16,131071,0.06038933495680491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,2,128,1,float16,fp8,131071,0.03558400024970373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,1,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,3,0.009418666362762451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,3,0.008229333286484083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,7,0.008074666683872541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,7,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,15,0.008309333274761835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,15,0.009418666362762451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,31,0.010197333370645842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,63,0.012138667205969492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,31,0.00919999989370505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,63,0.009381333366036415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,127,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,511,0.016528000434239704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,127,0.012063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,255,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,255,0.011978667229413986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,511,0.01646399994691213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,1023,0.0176959993938605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,2047,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,1023,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,2047,0.009312000125646591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,4095,0.010346666599313417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,4095,0.01570133368174235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,8191,0.019509332875410717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,16383,0.021738665799299877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,8191,0.012794667234023413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,16383,0.015135999768972397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,32767,0.02125866711139679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,32767,0.03474666674931844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,65535,0.04979733129342397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,65535,0.02714666724205017
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,1,0.009375999992092451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,1,0.008186666915814081
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,float16,131071,0.08283199866612752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,4,128,1,float16,fp8,131071,0.055786664287249245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,3,0.008229333286484083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,3,0.009413333609700203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,7,0.008309333274761835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,7,0.00938666673998038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,15,0.009599999835093817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,15,0.008672000219424566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,31,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,63,0.009349333122372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,127,0.009413333609700203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,63,0.012005332857370377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,127,0.011946666985750198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,255,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,255,0.012106666962305704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,511,0.016544000556071598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,511,0.016437333077192307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,31,0.010138666878143946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,1023,0.01351999988158544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,1023,0.008234666660428047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,4095,0.016384000579516094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,2047,0.015210667004187902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,2047,0.009119999905427298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,4095,0.010656000425418219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,8191,0.025583999852339428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,16383,0.03102933367093404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,8191,0.014127999544143677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,16383,0.016783999900023144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,32767,0.05012266834576925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,32767,0.03221333275238673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,65535,0.07786666850248973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,65535,0.04349866509437561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,1,0.008245333408315977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,float16,131071,0.13369066516558328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,16,8,128,1,float16,fp8,131071,0.0769760012626648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,1,0.009418666362762451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,3,0.008378666515151659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,3,0.009535999968647957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,7,0.009429333110650381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,7,0.008405333384871483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,15,0.008629333227872849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,15,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,31,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,63,0.012096000214417776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,31,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,63,0.009541333342591921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,127,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,127,0.012159999459981918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,255,0.010640000303586325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,511,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,255,0.012037333101034164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,511,0.01661866654952367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,1023,0.01812800019979477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,1023,0.01749333366751671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,4095,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,2047,0.02974933385848999
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,2047,0.02943466603755951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,4095,0.013295999417702356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,8191,0.016069332758585613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,8191,0.019909333437681198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,16383,0.019130667050679524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,16383,0.02117866774400075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,32767,0.025909334421157837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,32767,0.02441066751877467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,65535,0.03274133304754893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,65535,0.03105599929889043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,1,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,1,0.008256000156203905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,float16,131071,0.062463998794555664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,1,128,1,float16,fp8,131071,0.047184000412623085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,3,0.008303999900817871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,3,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,7,0.009429333110650381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,7,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,15,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,31,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,15,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,31,0.01022933361430963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,63,0.012037333101034164
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,255,0.01051733394463857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,63,0.009509333098928133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,127,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,127,0.012165332833925882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,511,0.016645333419243496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,255,0.01202133297920227
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,1023,0.0176959993938605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,511,0.016538667182127636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,1023,0.017605333278576534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,2047,0.015360000232855478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,2047,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,4095,0.016399999459584553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,8191,0.020106667031844456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,4095,0.010853332777818045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,8191,0.012981332838535309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,16383,0.022367998957633972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,16383,0.015370666980743408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,32767,0.021712000171343487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,32767,0.03658666710058848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,65535,0.050330668687820435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,65535,0.027514666318893433
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,float16,131071,0.08363733688990276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,1,0.008303999900817871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,1,0.009541333342591921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,3,0.009375999992092451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,3,0.008453333129485449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,2,128,1,float16,fp8,131071,0.043824002146720886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,7,0.008447999755541483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,7,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,31,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,15,0.008693333094318708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,31,0.010309333602587381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,15,0.00973866693675518
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,63,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,63,0.012351999680201212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,127,0.009712000067035357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,127,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,255,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,511,0.016597333053747814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,511,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,1023,0.013658666362365087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,2047,0.015861333658297855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,1023,0.008362666393319765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,4095,0.01710933322707812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,2047,0.009306666751702627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,4095,0.010794666906197866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,8191,0.02607999990383784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,16383,0.03200533241033554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,16383,0.017029333859682083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,8191,0.014250667144854864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,32767,0.05124799907207489
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,65535,0.0793333351612091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,32767,0.03236266722281774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,1,0.008389333263039589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,65535,0.04506133496761322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,float16,131071,0.13592533270517984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,4,128,1,float16,fp8,131071,0.07760533193747203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,1,0.009583999713261923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,3,0.008421333506703377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,3,0.009488000224033991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,7,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,7,0.008357333640257517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,15,0.008592000231146812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,31,0.010373333469033241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,31,0.009173333023985228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,63,0.009472000102202097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,127,0.012181332955757776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,63,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,255,0.01055466632048289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,127,0.009514666472872099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,511,0.01292266696691513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,2047,0.02073066681623459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,1023,0.01339200014869372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,1023,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,511,0.007738666608929634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,2047,0.010144000252087912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,4095,0.023978665471076965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,8191,0.035242666800816856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,8191,0.020794666061798733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,4095,0.01184533288081487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,16383,0.02587199956178665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,16383,0.05309333403905233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,32767,0.08053866525491078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,32767,0.04684799909591675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,65535,0.13262933492660522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,65535,0.075162669022878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,1,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,float16,131071,0.23147199551264444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,3,0.007893333211541176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,16,8,128,1,float16,fp8,131071,0.13707733154296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,3,0.010826667149861654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,1,0.008016000191370646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,7,0.010725333044926325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,7,0.007338666667540868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,15,0.010965333630641302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,15,0.007770666852593422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,31,0.010661333799362183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,31,0.00731733317176501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,63,0.008432000254591307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,63,0.010714666297038397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,127,0.010735999792814255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,255,0.013562666873137156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,127,0.008367999767263731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,255,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,511,0.0194560003777345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,511,0.011034666250149408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,1023,0.028805332879225414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,2047,0.04483733574549357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,1023,0.019866666446129482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,2047,0.027722666660944622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,4095,0.0697920024394989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,8191,0.09014933307965596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,4095,0.046069333950678505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,8191,0.06123200058937073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,float16,16383,0.13102400302886963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,1,0.010858666151762009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,3,0.011002667248249054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,1,0.006351999938488007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,1,128,1,float16,fp8,16383,0.08884800473848979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,3,0.006666666517655055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,7,0.01109333336353302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,15,0.006666666517655055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,7,0.006698666761318843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,63,0.010645333677530289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,31,0.0064853330453236895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,127,0.010677333921194077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,63,0.006400000303983688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,127,0.007125333572427432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,255,0.020981334149837494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,255,0.009152000149091085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,511,0.028197333216667175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,1023,0.04566933214664459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,511,0.01515199989080429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,2047,0.06621333460013072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,1023,0.02182399978240331
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,2047,0.0281333327293396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,4095,0.08611200253168742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,8191,0.12707199652989706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,4095,0.04308266441027323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,8191,0.06724800169467926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,float16,16383,0.21357333660125732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,2,128,1,float16,fp8,16383,0.1132480005423228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,3,0.015493333339691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,1,0.01571200042963028
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,1,0.0064106664309899015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,3,0.006586666529377301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,7,0.015765332927306492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,7,0.007045333584149678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,15,0.006858666737874349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,15,0.015557333827018738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,31,0.015658666690190632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,31,0.006549333532651265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,63,0.015295999745527903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,127,0.007573333258430163
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,63,0.006602666651209195
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,127,0.015594666202863058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,255,0.029893333713213604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,255,0.013823999712864557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,511,0.04637333254019419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,1023,0.06202666461467743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,511,0.024288001159826916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,1023,0.031701333820819855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,2047,0.08597866694132487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,2047,0.0447626660267512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,8191,0.21385065714518228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,4095,0.06786133348941803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,4095,0.12706666191418967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,8191,0.11384532848993938
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,1,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,float16,16383,0.3820426861445109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,1,0.025450666745503742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,3,0.025429333249727886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,3,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,7,0.025285333395004272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,4,128,1,float16,fp8,16383,0.2053226629892985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,7,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,15,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,15,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,63,0.009872000043590864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,31,0.025237334271272022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,31,0.009786666681369146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,63,0.02510933329661687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,127,0.02566933383544286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,255,0.023936000963052113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,255,0.04979733129342397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,511,0.06481599807739258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,511,0.03046400099992752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,1023,0.08283199866612752
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,2047,0.12868266304334006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,2047,0.06741333504517873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,1023,0.04422399898370107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,4095,0.21490132808685303
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,4095,0.11334932843844096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,8191,0.3827626705169678
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,8191,0.20502400398254395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,float16,16383,0.7194026311238607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,1,0.009306666751702627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,1,0.009488000224033991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,16,8,128,1,float16,fp8,16383,0.38710399468739826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,3,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,7,0.008400000010927519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,7,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,15,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,3,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,15,0.009722666814923286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,31,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,31,0.010602666685978571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,63,0.012138667205969492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,63,0.009343999748428663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,127,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,255,0.010672000547250112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,127,0.012159999459981918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,255,0.012304000556468964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,511,0.016629333297411602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,511,0.016837333639462788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,1023,0.02938666691382726
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,1023,0.02978666623433431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,2047,0.017082666357358296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,4095,0.01332266628742218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,2047,0.011653333902359009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,4095,0.017898666361967724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,8191,0.02162666618824005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,16383,0.024106666445732117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,8191,0.01611199975013733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,16383,0.019434666881958645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,32767,0.039359999199708305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,65535,0.054229333996772766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,32767,0.027717334528764088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,1,0.009514666472872099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,1,0.008527999743819237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,65535,0.03552533437808355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,float16,131071,0.08813866972923279
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,1,128,1,float16,fp8,131071,0.07178133229414622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,3,0.008527999743819237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,7,0.008752000207702318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,7,0.009514666472872099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,3,0.01051733394463857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,15,0.008736000085870424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,15,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,63,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,31,0.01055466632048289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,31,0.010410666465759277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,255,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,127,0.012282667060693106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,127,0.010575999816258749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,255,0.010666667173306147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,511,0.01682666689157486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,511,0.016666666915019352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,1023,0.008693333094318708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,1023,0.013536000003417334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,2047,0.017173333714405697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,4095,0.018602666755517323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,8191,0.014597332725922266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,2047,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,4095,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,8191,0.027402666707833607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,16383,0.03309333324432373
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,16383,0.017674667139848072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,32767,0.05372266471385956
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,32767,0.027456000447273254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,65535,0.05209066470464071
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,131071,0.1365386644999186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,float16,65535,0.08149866759777069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,1,0.008410666758815447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,3,0.008410666758815447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,2,128,1,float16,fp8,131071,0.07307200133800507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,1,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,3,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,7,0.008400000010927519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,31,0.009365333244204521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,7,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,15,0.00871999996403853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,63,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,31,0.010480000327030817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,63,0.012282667060693106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,127,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,511,0.01293333371480306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,255,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,511,0.007663999994595845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,255,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,1023,0.013541333377361298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,2047,0.010346666599313417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,1023,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,2047,0.021882665654023487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,8191,0.021087999145189922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,4095,0.01201066623131434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,4095,0.02536533276240031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,8191,0.03730133424202601
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,16383,0.05665066838264465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,16383,0.02632533262173335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,32767,0.08304533362388611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,32767,0.04749333361784617
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,65535,0.13569600383440653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,65535,0.07585066556930542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,1,0.00854399986565113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,fp8,131071,0.13823466499646506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,4,128,1,float16,float16,131071,0.23455999294916788
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,1,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,3,0.008463999877373377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,3,0.009733333562811216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,15,0.00873066671192646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,7,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,7,0.009648000200589498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,31,0.010474666953086853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,31,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,15,0.009877333417534828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,63,0.012266666938861212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,255,0.012485332787036896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,63,0.009648000200589498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,127,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,255,0.00731733317176501
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,511,0.013354666531085968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,1023,0.019237333287795384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,511,0.007861333588759104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,1023,0.009152000149091085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,2047,0.015103999525308609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,2047,0.028698667883872986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,4095,0.041296000281969704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,4095,0.017978666971127193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,8191,0.05906666815280914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,8191,0.02899733434120814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,16383,0.0476746658484141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,16383,0.08974933624267578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,32767,0.07810666660467784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,32767,0.1288320024808248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,65535,0.22013866901397705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,65535,0.13149866461753845
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,float16,131071,0.40646934509277344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,1,0.011136000355084738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,1,0.007189333438873291
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,7,0.011317333827416102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,3,0.011274666835864386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,3,0.007840000092983246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,16,8,128,1,float16,fp8,131071,0.22180799643198648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,7,0.007936000203092894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,15,0.007167999943097432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,31,0.011247999966144562
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,127,0.011226666470368704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,31,0.008112000301480293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,63,0.008080000057816505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,127,0.007882666463653246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,255,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,255,0.02143466720978419
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,511,0.018826667219400406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,511,0.02905600021282832
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,1023,0.047237331668535866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,2047,0.06801066795984904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,1023,0.033039999504884086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,2047,0.0454720010360082
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,4095,0.06101333101590475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,4095,0.08822932839393616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,float16,8191,0.1301866670449575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,3,0.007125333572427432
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,3,0.015834666788578033
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,1,0.01580799991885821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,1,0.007706666365265846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,1,128,1,float16,fp8,8191,0.08786666393280029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,7,0.01600533351302147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,7,0.007184000064929326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,15,0.015962666521469753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,15,0.007402666533986728
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,31,0.015685333559910457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,31,0.008026666939258575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,63,0.015882667154073715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,127,0.007893333211541176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,255,0.0305173322558403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,127,0.016389333953460056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,63,0.008101333553592363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,255,0.015253332753976187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,1023,0.06200533111890157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,511,0.021850667893886566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,1023,0.0284853329261144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,511,0.04682666560014089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,2047,0.08604266246159871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,2047,0.043151999513308205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,4095,0.12796266873677573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,4095,0.06698133548100789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,float16,8191,0.2159093419710795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,1,0.0252960001428922
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,1,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,7,0.02537599951028824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,2,128,1,float16,fp8,8191,0.11378133296966553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,3,0.025429333249727886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,31,0.025360000630219776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,15,0.025536000728607178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,7,0.010575999816258749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,15,0.010656000425418219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,63,0.010709332923094431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,127,0.02553066611289978
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,63,0.025386666258176167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,255,0.049423997600873314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,1023,0.08308800061543782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,255,0.023930666347344715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,511,0.0647680014371872
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,511,0.03139200061559677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,2047,0.12900267044703165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,1023,0.04527466495831808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,4095,0.2170613408088684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,2047,0.06832533578077953
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,4095,0.11493333180745442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,1,0.04473066826661428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,3,0.04465066889921824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,float16,8191,0.38783466815948486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,4,128,1,float16,fp8,8191,0.20725866158803305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,1,0.01655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,3,0.016677333662907284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,15,0.04446933170159658
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,7,0.016517333686351776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,7,0.04470933477083842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,63,0.01655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,31,0.044906665881474815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,63,0.04438399771849314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,31,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,15,0.016656000167131424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,127,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,255,0.05340266724427541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,127,0.01882133384545644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,1023,0.0627040018637975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,255,0.025626666843891144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,511,0.07264000177383423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,511,0.039359999199708305
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,2047,0.20498132705688477
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,1023,0.11425600449244182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,2047,0.1090666651725769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,1,0.01655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,4095,0.3738986651102702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,1,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,4095,0.20058667659759521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,3,0.00790933333337307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,float16,8191,0.7116533120473226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,16,8,128,1,float16,fp8,8191,0.3823946714401245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,3,0.01655999943614006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,7,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,15,0.007941333577036858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,7,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,15,0.016688000410795212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,31,0.016458666572968166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,31,0.008826666822036108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,63,0.00790933333337307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,127,0.016362667083740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,63,0.017551999539136887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,1023,0.0641546646753947
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,255,0.0314026673634847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,127,0.009866666669646898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,511,0.03202133377393087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,255,0.0180479995906353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,511,0.048911998669306435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,1023,0.045653333266576133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,float16,2047,0.08839999636014302
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,1,0.010543999572594961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,1,0.026560001075267792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,3,0.025957333544890087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,1,128,1,float16,fp8,2047,0.060458665092786155
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,7,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,7,0.026026666164398193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,15,0.02638400097688039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,3,0.011642667154471079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,63,0.025813333690166473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,15,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,31,0.025989333788553875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,63,0.010591999938090643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,31,0.010570666442314783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,127,0.026288000245889027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,255,0.050288001696268715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,511,0.02957333376010259
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,255,0.022074667116006214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,511,0.06562666594982147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,1023,0.043141335248947144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,127,0.012448000411192576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,1023,0.08441600203514099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,float16,2047,0.13174399733543396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,2,128,1,float16,fp8,2047,0.06761066615581512
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,1,0.045328001181284584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,1,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,3,0.0450133333603541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,7,0.016549333930015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,7,0.04490133126576742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,15,0.04474133253097534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,31,0.045109331607818604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,15,0.016549333930015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,3,0.01647466669480006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,31,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,63,0.044250667095184326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,63,0.016554666062196095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,127,0.04557866851488749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,127,0.019402666638294857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,255,0.053770666321118675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,511,0.040709334115187325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,511,0.07340266803900401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,1023,0.11693867047627766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,255,0.027119999130566914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,1023,0.06341866652170818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,float16,2047,0.20946133136749268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,1,0.08133333424727122
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,7,0.08153066535790761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,4,128,1,float16,fp8,2047,0.1097866694132487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,1,0.030266667405764263
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,3,0.08161599934101105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,3,0.030031998952229817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,7,0.03002133220434189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,15,0.030031998952229817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,15,0.08123733103275299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,31,0.08109866579373677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,31,0.030048000315825146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,127,0.08265600105126698
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,63,0.08096000055472057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,127,0.036576000352700554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,255,0.09492266178131104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,63,0.030031998952229817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,511,0.073594664533933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,255,0.05090666810671488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,511,0.12863999605178833
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,1023,0.11979200442632039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,1023,0.2095573345820109
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,float16,2047,0.38501866658528644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,1,0.027114666998386383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,3,0.013023999830087027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,16,8,128,1,float16,fp8,2047,0.21227733294169107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,1,0.012863999853531519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,7,0.013050666699806849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,3,0.027813332776228588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,7,0.02740799884001414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,15,0.027317332724730175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,15,0.012986666212479273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,63,0.027024000883102417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,31,0.012879999975363413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,63,0.012885333349307379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,31,0.02712533374627431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,127,0.015125333021084467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,255,0.05273599922657013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,127,0.02719466636578242
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,255,0.03200533241033554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,1023,0.06124266485373179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,1,0.04602666695912679
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,511,0.0677706648906072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,fp8,511,0.04710933566093445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,3,0.04587733248869578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,1,128,1,float16,float16,1023,0.08733333150545756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,3,0.015189333508412043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,7,0.04619200030962626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,31,0.04613333443800608
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,15,0.046336000164349876
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,15,0.014959999670584997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,63,0.045706664522488914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,63,0.014949332922697067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,127,0.047653332352638245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,127,0.01801066721479098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,31,0.01533866673707962
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,255,0.05527466535568237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,511,0.07534933090209961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,511,0.04066666712363561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,255,0.02604266752799352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,float16,1023,0.11844266454378764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,2,128,1,float16,fp8,1023,0.06397866706053416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,1,0.03009066730737686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,3,0.08163733283678691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,3,0.03009066730737686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,7,0.02998399982849757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,7,0.08189333478609721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,15,0.08197333415349324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,1,0.08207466701666515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,31,0.08137066662311554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,63,0.08218133449554443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,31,0.03010133405526479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,15,0.030080000559488933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,127,0.08356266220410664
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,63,0.02993600070476532
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,255,0.09582933783531189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,127,0.0391146664818128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,255,0.05234666665395101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,511,0.07550399998823802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,511,0.13179733355840048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,float16,1023,0.21248533328374228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,1,0.15059733390808105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,3,0.15080533425013223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,4,128,1,float16,fp8,1023,0.12125333150227864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,3,0.06305066744486491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,1,0.06298666695753734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,7,0.1506239970525106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,7,0.0626933326323827
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,15,0.15019733707110086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,15,0.06311999758084615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,31,0.15314666430155435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,63,0.0631466656923294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,31,0.06310933331648509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,127,0.15389866630236307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,127,0.07262399792671204
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,63,0.15333333611488342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,255,0.17655466000239053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,511,0.24225600560506186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,255,0.09449600179990132
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,1023,0.23149865865707397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,fp8,511,0.1400373379389445
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,16,8,128,1,float16,float16,1023,0.3954933484395345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,1,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,3,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,1,0.010762666662534079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,7,0.008570666735370954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,3,0.010634666929642359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,15,0.008821333448092142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,31,0.009392000113924345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,7,0.010645333677530289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,15,0.010101333260536194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,63,0.009519999846816063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,31,0.011477333803971609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,127,0.012234666695197424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,63,0.012373333175977072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,127,0.009770666559537252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,255,0.010741333166758219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,511,0.028255999088287354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,255,0.012282667060693106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,1023,0.013679999858140945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,511,0.027999999622503918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,2047,0.01749333366751671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,4095,0.018496000518401463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,1023,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,2047,0.011733333269755045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,8191,0.02762666592995326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,4095,0.013530666629473368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,16383,0.034373333056767784
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,16383,0.02271999915440877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,8191,0.018837332725524902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,32767,0.056218668818473816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,32767,0.04334400097529093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,65535,0.08357333143552144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,65535,0.05733866492907206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,1,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,fp8,131071,0.10039466619491577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,1,0.010581333190202713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,1,128,1,float16,float16,131071,0.13902399937311807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,3,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,3,0.009525333220760027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,7,0.009824000298976898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,15,0.009061333412925402
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,31,0.009466666728258133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,31,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,63,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,127,0.012351999680201212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,255,0.010703999549150467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,63,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,255,0.012245333443085352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,127,0.010805333654085795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,511,0.012629333883523941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,1023,0.013290667285521826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,511,0.00895999992887179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,1023,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,2047,0.01102399950226148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,2047,0.022437334060668945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,4095,0.025631998976071674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,4095,0.012613333761692047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,8191,0.03659199923276901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,16383,0.055493334929148354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,16383,0.029525332152843475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,8191,0.017759999881188076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,32767,0.08354666829109192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,32767,0.04403733213742574
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,65535,0.0751093327999115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,65535,0.137061337629954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,1,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,float16,131071,0.23568532864252725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,2,128,1,float16,fp8,131071,0.1188106636206309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,1,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,3,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,7,0.008613333106040955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,3,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,15,0.008890666688481966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,7,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,15,0.009952000031868616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,31,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,63,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,31,0.010330666477481524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,127,0.009839999799927076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,255,0.0124746672809124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,63,0.01239466667175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,255,0.007381333038210869
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,511,0.013199999928474426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,1023,0.019088000059127808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,511,0.008912000184257826
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,1023,0.009285333255926767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,2047,0.028730665644009907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,4095,0.0417546679576238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,4095,0.017984000345071156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,8191,0.05955199897289276
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,16383,0.09058133761088054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,16383,0.047781333327293396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,8191,0.029109333952267964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,32767,0.07776533563931783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,32767,0.13088533282279968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,65535,0.22140800952911377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,65535,0.13205333550771078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,1,0.010197333370645842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,1,0.006400000303983688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,float16,131071,0.4082506497701009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,4,128,1,float16,fp8,131071,0.22380266586939493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,7,0.010181333248813948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,3,0.006090666477878888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,3,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,7,0.006229333579540253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,31,0.006208000083764394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,15,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,15,0.0061919999619325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,31,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,63,0.010426666587591171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,63,0.00613866684337457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,127,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,127,0.00660800002515316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,255,0.01341333364446958
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,255,0.008250666782259941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,511,0.019317333896954853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,1023,0.028202667832374573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,511,0.008714666590094566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,1023,0.015178666760524115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,2047,0.04369066655635834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,8191,0.08737599849700928
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,4095,0.06817600131034851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,4095,0.03249066571394602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,2047,0.02067199970285098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,8191,0.04678933322429657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,16383,0.12619200348854065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,16383,0.06967466572920482
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,32767,0.21236799160639444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,32767,0.11663466691970825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,65535,0.38154133160909015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,65535,0.209498663743337
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,float16,131071,0.721781333287557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,1,0.048021331429481506
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,1,0.02276266614596049
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,3,0.04818666477998098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,16,8,128,1,float16,fp8,131071,0.3949226538340251
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,7,0.047930667797724404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,3,0.02271999915440877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,7,0.0227360005180041
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,15,0.04781333108743032
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,15,0.02269333352645238
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,31,0.047824000318845115
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,63,0.04755199948946635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,31,0.022704000274340313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,127,0.04956266780694326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,63,0.022624000906944275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,127,0.02717866748571396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,255,0.05890133480230967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,255,0.041989331444104515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,1,0.08358400066693623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,float16,511,0.07835199932257335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,3,0.08385599652926128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,1,128,1,float16,fp8,511,0.056234667698542275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,3,0.026357332865397137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,1,0.02659733345111211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,7,0.08364799618721008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,7,0.02638400097688039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,15,0.02649066597223282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,15,0.08366400003433228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,31,0.08354133367538452
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,31,0.02649066597223282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,63,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,63,0.08486933509508769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,127,0.08619200189908345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,127,0.037434667348861694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,255,0.09988266229629517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,255,0.04887466629346212
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,float16,511,0.1355946660041809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,2,128,1,float16,fp8,511,0.07051733136177063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,1,0.0655680000782013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,3,0.1514293352762858
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,1,0.15135467052459717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,3,0.066021333138148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,7,0.15155200163523355
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,7,0.06567466755708058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,15,0.15189866224924722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,15,0.06549333532651265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,31,0.15504533052444458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,63,0.15545066197713217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,127,0.15497066577275595
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,31,0.06573866804440816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,63,0.0662666658560435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,127,0.07601066430409749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,255,0.17999466260274252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,255,0.09796800216039021
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,float16,511,0.2475093404452006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,4,128,1,float16,fp8,511,0.1439893345038096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,1,0.2908800045649211
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,1,0.12412800391515096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,3,0.2903093298276265
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,3,0.1242026686668396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,7,0.2918506662050883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,7,0.12437867124875386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,15,0.29502934217453003
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,15,0.12404800454775493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,31,0.12428266803423564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,31,0.29844266176223755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,63,0.2982719937960307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,63,0.12433600425720215
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,127,0.29639999071757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,127,0.13794666528701782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,255,0.3419040044148763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,255,0.18094400564829508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,fp8,511,0.27084799607594806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,1,0.08803733189900716
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,1,0.04805333415667216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,3,0.08783466617266338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,16,8,128,1,float16,float16,511,0.4692373275756836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,3,0.04844800134499868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,7,0.08795733253161113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,7,0.04706666866938273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,15,0.08850133419036865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,31,0.08854400118192036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,15,0.048197334011395775
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,63,0.08973866701126099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,31,0.04821333289146423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,63,0.04804799954096476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,127,0.09016000231107076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,127,0.06777599950631459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,float16,255,0.10548800230026245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,1,0.15559466679890951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,1,128,1,float16,fp8,255,0.08050133287906647
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,1,0.05750399827957153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,3,0.05735466877619425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,7,0.1564640005429586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,7,0.05746666590372721
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,3,0.15571733315785727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,15,0.15732799967130026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,15,0.0577706644932429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,31,0.05738133192062378
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,63,0.16177067160606384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,63,0.05715199808279673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,31,0.15969066818555197
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,127,0.0678773323694865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,255,0.18726933002471924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,float16,127,0.16159466902414957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,1,0.12626133362452188
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,3,0.29448533058166504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,2,128,1,float16,fp8,255,0.0869653324286143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,3,0.12586133678754172
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,1,0.29367466767628986
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,7,0.29604800542195636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,15,0.300656000773112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,15,0.12615999579429626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,31,0.30482665697733563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,31,0.12615999579429626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,7,0.12611200412114462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,63,0.3035200039545695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,63,0.1272426644961039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,127,0.2989120086034139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,127,0.14421332875887552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,float16,255,0.34912534554799396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,4,128,1,float16,fp8,255,0.18810667594273886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,1,0.573861320813497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,3,0.24154667059580484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,1,0.241482675075531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,3,0.5749386548995972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,7,0.5812319914499918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,7,0.24172266324361166
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,15,0.5864426692326864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,31,0.5878880023956299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,63,0.24276266495386759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,15,0.2413813273111979
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,31,0.24134933948516846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,63,0.583845337231954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,127,0.5817333459854126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,127,0.26863465706507367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,1,0.010117333382368088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,float16,255,0.6702613035837809
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,1,0.014090667168299357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,3,0.009141333401203156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,3,0.014106666048367819
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,7,0.014085333794355392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,16,8,128,1,float16,fp8,255,0.35396798451741535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,7,0.010234666367371878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,31,0.015791999797026317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,15,0.014538666854302088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,63,0.01590399940808614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,31,0.011823999385039011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,63,0.01184533288081487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,127,0.011882666498422623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,255,0.018453333526849747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,255,0.013584000368913015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,127,0.016095999628305435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,511,0.012885333349307379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,1023,0.013605333864688873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,511,0.010101333260536194
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,2047,0.022517333428064983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,1023,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,2047,0.013386666774749756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,4095,0.02586666742960612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,4095,0.015471999843915304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,8191,0.03806933263937632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,8191,0.02733866622050603
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,16383,0.057349334160486855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,16383,0.03623999903599421
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,32767,0.0865280032157898
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,32767,0.06213866670926412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,1,0.009018666421373686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,1,0.014010666559139887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,3,0.009082666908701261
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,fp8,65535,0.09697600205739339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,3,0.014101333916187286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,1,128,1,float16,float16,65535,0.13763733704884848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,7,0.009183999771873156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,7,0.014256000518798828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,15,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,15,0.014346666634082794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,31,0.01569066693385442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,31,0.011855999628702799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,63,0.011877333124478659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,63,0.016021333634853363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,127,0.011749333391586939
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,127,0.01618133361140887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,255,0.007893333211541176
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,511,0.013130666067202887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,255,0.012890666723251343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,511,0.008378666515151659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,1023,0.01002133327225844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,1023,0.019178666174411774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,8191,0.05961066484451294
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,2047,0.012906666845083237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,4095,0.0423573354880015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,2047,0.029205332199732464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,4095,0.019546666493018467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,8191,0.027045334378878277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,16383,0.04553600152333578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,32767,0.06942933301130931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,32767,0.1304746667544047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,16383,0.09039466579755147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,fp8,65535,0.1139413317044576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,1,0.010437333335479101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,2,128,1,float16,float16,65535,0.2204586664835612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,1,0.006122666721542676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,3,0.006330666442712148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,3,0.010304000228643417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,7,0.006506666541099548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,15,0.006351999938488007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,15,0.010533332824707031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,7,0.011338666081428528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,31,0.011477333803971609
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,31,0.006213333457708359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,127,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,63,0.01007466639081637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,63,0.005893333504597346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,127,0.006480000292261441
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,255,0.01314666618903478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,255,0.008698666468262672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,511,0.008682666967312494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,1023,0.015439999600251516
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,2047,0.043418665726979576
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,511,0.019237333287795384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,1023,0.027935999135176342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,4095,0.03260799994071325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,2047,0.020831999679406483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,4095,0.06781866649786632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,8191,0.08725866675376892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,8191,0.046949331959088646
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,16383,0.06971733272075653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,16383,0.12823999921480814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,32767,0.11639466881752014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,32767,0.21317332983016968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,1,0.01044800008336703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,float16,65535,0.3832586606343587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,4,128,1,float16,fp8,65535,0.2092746694882711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,3,0.010672000547250112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,1,0.006208000083764394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,3,0.006437333300709724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,7,0.010666667173306147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,7,0.006224000205596288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,15,0.010591999938090643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,31,0.010709332923094431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,15,0.006234666953484218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,31,0.006106666599710782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,63,0.006954666847983996
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,63,0.01032533310353756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,255,0.021141332884629566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,127,0.006720000257094701
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,127,0.010741333166758219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,255,0.008383999889095625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,511,0.028362666567166645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,511,0.014181333283583323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,1023,0.04587199787298838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,1023,0.024336000283559162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,2047,0.030432000756263733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,2047,0.06670400003592174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,4095,0.08613866567611694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,4095,0.04452799757321676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,8191,0.12473600109418233
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,16383,0.20995734135309854
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,8191,0.06746666630109151
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,16383,0.11301866173744202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,32767,0.3784799973169963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,32767,0.2042293349901835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,1,0.01403733342885971
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,float16,65535,0.715002695719401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,1,0.009130666653315226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,16,8,128,1,float16,fp8,65535,0.3863946596781413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,3,0.014149333039919535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,3,0.009216000015536943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,7,0.009119999905427298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,7,0.01402666668097178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,15,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,15,0.014560000350077948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,31,0.015770666301250458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,63,0.01594666639963786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,31,0.011957333733638128
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,63,0.01184533288081487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,127,0.016229332735141117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,127,0.011877333124478659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,511,0.01322666679819425
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,255,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,255,0.012506666282812754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,511,0.008298666526873907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,1023,0.019130667050679524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,1023,0.009973333527644476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,2047,0.012768000364303589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,2047,0.029232000311215717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,4095,0.04168533285458883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,4095,0.019695999721686046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,8191,0.06092800199985504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,8191,0.026714667677879333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,16383,0.09215466181437175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,16383,0.04612799982229868
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,32767,0.13140799601872763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,32767,0.06856533388296764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,1,0.01027199998497963
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,float16,65535,0.22091732422510782
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,1,128,1,float16,fp8,65535,0.11372266213099162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,3,0.010368000095089277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,1,0.005999999741713206
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,7,0.010384000216921171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,3,0.006394666930039723
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,7,0.006058666855096817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,31,0.01032533310353756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,15,0.010437333335479101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,15,0.006085333103934924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,31,0.006261333202322324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,63,0.010234666367371878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,63,0.006101333225766818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,127,0.010362666721145311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,127,0.006784000123540561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,255,0.01368533323208491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,511,0.019199999670187633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,255,0.007610666876037915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,511,0.008634666601816813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,1023,0.027797333896160126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,1023,0.015354666858911514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,2047,0.044400001565615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,2047,0.020986666282018025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,4095,0.06769600013891856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,4095,0.032826667030652366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,8191,0.08786666393280029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,8191,0.04689066608746847
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,16383,0.12846400340398154
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,16383,0.07009600102901459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,32767,0.21241599321365356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,32767,0.11638399958610535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,1,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,1,0.0061493335912625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,fp8,65535,0.21098132928212485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,3,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,2,128,1,float16,float16,65535,0.38130664825439453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,3,0.006101333225766818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,7,0.006415999804933866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,7,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,15,0.010661333799362183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,15,0.006277333324154218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,31,0.010543999572594961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,31,0.006229333579540253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,63,0.006495999793211619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,63,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,127,0.010687999427318573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,127,0.006597333277265231
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,255,0.008336000144481659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,255,0.020576000213623047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,511,0.028101332485675812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,511,0.014175999909639359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,1023,0.04611733555793762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,1023,0.024298667907714844
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,2047,0.06618133187294006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,2047,0.030581332743167877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,4095,0.08555733164151509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,4095,0.04506133496761322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,8191,0.1262453297773997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,8191,0.06765866776307423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,16383,0.21225599447886148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,16383,0.11377066373825073
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,32767,0.379258672396342
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,32767,0.20470933119455972
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,1,0.008074666683872541
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,1,0.009162666896979014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,3,0.008341333518425623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,float16,65535,0.7171786626180013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,8,4,128,1,float16,fp8,65535,0.3863840103149414
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,3,0.009343999748428663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,7,0.008410666758815447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,7,0.009343999748428663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,15,0.008303999900817871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,15,0.009402666861812273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,31,0.010170666500926018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,63,0.009237333511312803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,31,0.009119999905427298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,63,0.01192533348997434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,127,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,127,0.011776000261306763
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,255,0.012053333222866058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,255,0.0102613332370917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,511,0.01635733370979627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,1023,0.017338667064905167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,1023,0.017386666188637417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,511,0.01635733370979627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,2047,0.017418666432301205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,2047,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,4095,0.017722666263580322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,4095,0.017632000148296356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,8191,0.012879999975363413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,16383,0.020207999895016353
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,8191,0.018415999909241993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,16383,0.015200000256299973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,32767,0.019098666807015736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,32767,0.023898666103680927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,65535,0.029093332588672638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,65535,0.024106666445732117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,1,0.008383999889095625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,fp8,131071,0.03186133255561193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,1,128,1,float16,float16,131071,0.03976000100374222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,1,0.009349333122372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,3,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,3,0.009301333377758661
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,7,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,7,0.00915733352303505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,15,0.008314666648705801
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,15,0.00938666673998038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,63,0.009306666751702627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,31,0.010048000141978264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,31,0.009178666397929192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,63,0.01191466674208641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,127,0.00921066664159298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,127,0.011930666863918304
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,255,0.01033599985142549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,255,0.011909333368142446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,511,0.016330666840076447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,511,0.01634666696190834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,1023,0.01743999992807706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,1023,0.017258666455745697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,2047,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,4095,0.015376000354687372
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,2047,0.01759999990463257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,4095,0.010549332946538925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,8191,0.018863999595244724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,8191,0.01251199965675672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,16383,0.020549333343903225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,16383,0.014869333555301031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,32767,0.025397333006064098
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,32767,0.018960000326236088
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,65535,0.03154666721820831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,65535,0.02404266595840454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,float16,131071,0.060565332571665444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,1,0.008080000057816505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,1,0.009162666896979014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,2,128,1,float16,fp8,131071,0.035301332672437034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,3,0.008277333031098047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,7,0.00814933329820633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,3,0.009066666786869368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,7,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,15,0.008517333616813024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,15,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,31,0.009189333145817121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,31,0.010144000252087912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,63,0.01190399999419848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,63,0.009322666873534521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,127,0.011999999483426413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,127,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,255,0.012005332857370377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,255,0.010309333602587381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,511,0.016528000434239704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,1023,0.01738133281469345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,511,0.016197333733240765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,2047,0.009141333401203156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,1023,0.017375999440749485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,4095,0.01581866666674614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,4095,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,8191,0.019541333119074505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,8191,0.012421333541472753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,16383,0.021744000415007275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,16383,0.015157333264748255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,32767,0.035258665680885315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,32767,0.02107200026512146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,65535,0.04939733445644379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,65535,0.026949333647886913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,1,0.008127999802430471
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,1,0.00915733352303505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,3,0.008298666526873907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,float16,131071,0.08331733445326488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,8,4,128,1,float16,fp8,131071,0.0553653339544932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,3,0.009472000102202097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,7,0.008474666625261307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,7,0.009279999881982803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,15,0.008469333251317343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,15,0.009477333476146063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,31,0.010255999863147736
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,31,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,63,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,127,0.00922133338948091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,63,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,127,0.011936000237862269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,255,0.010378666842977205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,255,0.012085333466529846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,511,0.01647466669480006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,511,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,1023,0.017605333278576534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,1023,0.01749333366751671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,2047,0.017573333034912746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,2047,0.017674667139848072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,4095,0.01637866720557213
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,4095,0.010709332923094431
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,8191,0.019610666980346043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,8191,0.012810666114091873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,16383,0.02117866774400075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,16383,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,32767,0.019402666638294857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,32767,0.025477332373460133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,65535,0.032218667368094124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,1,0.008197333042820295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,1,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,65535,0.024175999065240223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,3,0.008336000144481659
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,float16,131071,0.062405332922935486
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,1,128,1,float16,fp8,131071,0.03584533433119456
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,3,0.00921066664159298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,7,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,7,0.009237333511312803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,15,0.008410666758815447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,15,0.009493333597977957
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,31,0.009290666629870733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,31,0.010330666477481524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,63,0.009253333633144697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,63,0.012053333222866058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,127,0.00927466650803884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,255,0.010378666842977205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,127,0.011962667107582092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,255,0.012053333222866058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,511,0.016714667280515034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,1023,0.017637333522240322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,511,0.016528000434239704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,1023,0.017258666455745697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,2047,0.01581866666674614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,2047,0.009098666409651438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,4095,0.01634666696190834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,4095,0.010384000216921171
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,8191,0.019920000185569126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,16383,0.022405333817005157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,8191,0.012613333761692047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,16383,0.015141333142916361
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,32767,0.021327999730904896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,32767,0.03579200059175491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,65535,0.05085866649945577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,65535,0.027301333844661713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,1,0.008197333042820295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,1,0.009226666763424873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,float16,131071,0.08273066580295563
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,2,128,1,float16,fp8,131071,0.05637866755326589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,3,0.008346666892369589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,3,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,7,0.008181333541870117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,7,0.009904000287254652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,15,0.008410666758815447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,15,0.009648000200589498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,31,0.009088000282645226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,31,0.01044800008336703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,63,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,63,0.012074666718641916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,127,0.00922133338948091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,127,0.012074666718641916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,255,0.010480000327030817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,255,0.011968000481526056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,511,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,511,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,1023,0.013477332890033722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,1023,0.008234666660428047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,2047,0.009136000027259191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,2047,0.01584533353646596
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,4095,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,4095,0.017258666455745697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,8191,0.026309333741664886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,8191,0.014192000031471252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,16383,0.03182933231194814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,16383,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,32767,0.052282666166623436
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,32767,0.032357332607110344
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,65535,0.08030400176843007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,65535,0.044490665197372437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,float16,131071,0.13557333747545877
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,1,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,1,0.006442666674653689
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,3,0.01062400018175443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,8,4,128,1,float16,fp8,131071,0.07714133461316426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,3,0.0080960001796484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,7,0.01055466632048289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,15,0.010405333091815313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,7,0.006533333410819371
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,15,0.0064266665528217954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,31,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,31,0.0069226666043202085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,63,0.010741333166758219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,127,0.01044800008336703
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,63,0.006464000170429547
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,127,0.008005333443482717
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,255,0.013855999956528345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,255,0.007962666451931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,511,0.01932266727089882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,511,0.009477333476146063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,1023,0.027978666126728058
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,1023,0.015354666858911514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,2047,0.043824002146720886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,2047,0.018863999595244724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,4095,0.06856533388296764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,4095,0.03047466774781545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,8191,0.08930133779843648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,8191,0.04387733340263367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,16383,0.12945600350697836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,1,0.010842667271693548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,16383,0.06769600013891856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,1,0.00625599982837836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,float16,32767,0.21451733509699503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,1,128,1,float16,fp8,32767,0.11461866895357768
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,3,0.011600000162919363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,3,0.006277333324154218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,7,0.01080000028014183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,7,0.008405333384871483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,31,0.01108266661564509
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,15,0.0069973332186539965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,63,0.010543999572594961
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,31,0.012506666282812754
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,63,0.006239999706546466
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,127,0.010735999792814255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,511,0.028016000986099243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,255,0.020666666328907013
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,255,0.008362666393319765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,511,0.01423466702302297
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,1023,0.04624533156553904
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,2047,0.06619733572006226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,1023,0.02460266649723053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,2047,0.031557333966096245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,4095,0.08615466952323914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,4095,0.045408000548680626
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,8191,0.06846933563550313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,8191,0.1267253359158834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,16383,0.21316800514856973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,16383,0.11458667119344075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,1,0.015568000574906668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,float16,32767,0.382970650990804
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,2,128,1,float16,fp8,32767,0.20559465885162354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,1,0.006704000135262807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,3,0.015418666104475657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,3,0.006906666482488315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,7,0.015802666544914246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,7,0.006874666859706243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,15,0.006511999915043513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,15,0.015477333217859268
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,31,0.015717333803574245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,63,0.015295999745527903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,31,0.008517333616813024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,127,0.01573333392540614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,63,0.006735999758044879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,127,0.007221333061655362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,255,0.0301706666747729
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,255,0.013674666484196981
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,511,0.046613335609436035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,1023,0.062021334966023765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,511,0.023770667612552643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,1023,0.030080000559488933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,2047,0.08549867073694865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,2047,0.044549331068992615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,4095,0.1267680029074351
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,4095,0.0674720009167989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,8191,0.21244800090789795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,8191,0.11329066753387451
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,1,0.008293333152929941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,16383,0.3818133274714152
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,16383,0.2042186657587687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,1,0.009434666484594345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,fp8,32767,0.3869280020395915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,8,4,128,1,float16,float16,32767,0.7178773085276285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,3,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,3,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,7,0.008362666393319765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,15,0.008517333616813024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,7,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,15,0.009775999933481216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,31,0.009488000224033991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,31,0.010144000252087912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,63,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,127,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,63,0.012240000069141388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,127,0.012128000458081564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,255,0.010768000036478043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,255,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,511,0.016650666793187458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,1023,0.017637333522240322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,511,0.01658133293191592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,1023,0.017498667041460674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,2047,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,2047,0.017535999417304993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,8191,0.021557333568731945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,4095,0.010784000158309937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,4095,0.017765333255132038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,8191,0.013072000195582708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,16383,0.01544533297419548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,16383,0.023413332800070446
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,32767,0.03775466730197271
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,32767,0.02184533327817917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,65535,0.053674668073654175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,65535,0.02784000088771184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,1,0.008352000266313553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,float16,131071,0.08574933807055156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,1,0.009568000212311745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,1,128,1,float16,fp8,131071,0.04364799956480662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,3,0.008469333251317343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,3,0.009573333586255709
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,7,0.008549333239595095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,15,0.008687999720374743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,15,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,7,0.00955200009047985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,31,0.01028266673286756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,31,0.009408000235756239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,63,0.009365333244204521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,63,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,127,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,255,0.012159999459981918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,127,0.009328000247478485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,255,0.010464000205198923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,511,0.01670933390657107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,511,0.016597333053747814
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,2047,0.01687466725707054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,1023,0.013397333522637686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,1023,0.008341333518425623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,2047,0.009258666386206945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,4095,0.010597333312034607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,4095,0.018160000443458557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,8191,0.02739199995994568
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,8191,0.014256000518798828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,16383,0.033359999457995095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,16383,0.0169813334941864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,32767,0.05505600074927012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,32767,0.032074667513370514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,65535,0.044810667634010315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,65535,0.08140266438325246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,float16,131071,0.1368000010649363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,1,0.009519999846816063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,1,0.0085333331177632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,3,0.00842666688064734
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,3,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,2,128,1,float16,fp8,131071,0.07810133198897044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,7,0.008394666636983553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,7,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,31,0.00922133338948091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,15,0.008650666723648706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,15,0.009888000165422758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,31,0.011450666934251785
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,63,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,63,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,127,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,255,0.010575999816258749
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,255,0.012335999558369318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,511,0.012885333349307379
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,511,0.00786666696270307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,1023,0.008261333530147871
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,1023,0.013557333499193192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,2047,0.022181332111358643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,2047,0.01020800011853377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,4095,0.025514667232831318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,8191,0.03642133375008901
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,4095,0.011855999628702799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,8191,0.02107733239730199
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,16383,0.05643199880917867
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,16383,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,32767,0.08356799681981404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,32767,0.04666133224964142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,65535,0.1358453333377838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,float16,131071,0.23541333278020224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,65535,0.07534400125344594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,1,0.010944000134865442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,1,0.006565333033601443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,8,4,128,1,float16,fp8,131071,0.1376426617304484
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,3,0.010847999403874079
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,3,0.006400000303983688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,7,0.01097600037852923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,15,0.01090666651725769
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,15,0.006400000303983688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,7,0.008506666868925095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,31,0.010874666273593903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,31,0.006453333422541618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,63,0.010757333288590113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,63,0.006682666639486949
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,127,0.009472000102202097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,255,0.02110933264096578
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,255,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,511,0.028149334092934925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,1023,0.0459199994802475
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,511,0.015029333531856537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,1023,0.02181866765022278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,2047,0.06644799808661143
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,2047,0.02977066735426585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,4095,0.08716266353925069
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,4095,0.043103997906049095
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,8191,0.1281760036945343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,8191,0.06700799862543742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,float16,16383,0.21491734186808267
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,1,128,1,float16,fp8,16383,0.1136853297551473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,1,0.006762666627764702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,3,0.0068853336075941724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,3,0.015637333194414776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,1,0.015365333606799444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,7,0.015498666713635126
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,7,0.006634666894872983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,31,0.015493333339691162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,15,0.007061333085099856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,15,0.015573333948850632
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,63,0.015370666980743408
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,31,0.008165333420038223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,63,0.009962666779756546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,127,0.015647999942302704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,127,0.007461333026488622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,255,0.030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,255,0.01381333296497663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,511,0.04646400113900503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,511,0.024218666056791942
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,1023,0.06197333335876465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,1023,0.0313226655125618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,2047,0.08753599723180135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,2047,0.045093332727750145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,4095,0.1279306709766388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,4095,0.06830400228500366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,1,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,8191,0.11435733238855998
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,1,0.025610665480295818
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,8191,0.21394666035970053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,float16,16383,0.3856319983800252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,2,128,1,float16,fp8,16383,0.206496000289917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,3,0.025285333395004272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,3,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,7,0.025701334079106648
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,15,0.02552533398071925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,31,0.025407999753952026
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,15,0.01003200002014637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,63,0.02493866781393687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,7,0.010703999549150467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,31,0.00980266680320104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,63,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,127,0.025663999219735462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,255,0.049322664737701416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,255,0.023562667270501454
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,511,0.030479999879995983
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,511,0.06465066472689311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,1023,0.08335999647776286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,2047,0.129013329744339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,1023,0.0444213350613912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,4095,0.2162239948908488
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,2047,0.06726400057474773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,4095,0.11390399932861328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,8191,0.38651732603708905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,1,0.015909332782030106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,3,0.016000000139077503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,1,0.007760000104705493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,8191,0.2057173252105713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,float16,16383,0.7263733545939127
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,7,0.015989333391189575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,8,4,128,1,float16,fp8,16383,0.3874239921569824
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,7,0.00721066693464915
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,3,0.008080000057816505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,15,0.015882667154073715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,15,0.007834666719039282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,31,0.01600533351302147
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,31,0.007242666557431221
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,63,0.01575999955336253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,63,0.007765333478649457
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,127,0.016122666498025257
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,127,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,255,0.014896000425020853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,255,0.030400000512599945
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,511,0.04693866769472758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,511,0.021770666042963665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,1023,0.06229333579540253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,1023,0.029616000751654308
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,2047,0.04304533203442892
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,1,0.025450666745503742
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,fp8,4095,0.06679999828338623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,2047,0.08727467060089111
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,1,128,1,float16,float16,4095,0.1288479963938395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,3,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,1,0.010512000570694605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,3,0.025663999219735462
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,7,0.02532266577084859
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,7,0.00984533317387104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,15,0.025621332228183746
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,15,0.009829333052039146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,31,0.025402667621771496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,63,0.025434667865435284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,63,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,31,0.010496000448862711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,127,0.025706666211287182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,255,0.02399466683467229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,255,0.05013333261013031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,511,0.06461333235104878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,511,0.031888000667095184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,1023,0.04557333389918009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,1023,0.08344533046086629
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,2047,0.06831466654936473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,4095,0.21967466672261557
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,float16,2047,0.13107200463612875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,1,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,2,128,1,float16,fp8,4095,0.11508799592653911
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,1,0.04444266855716705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,3,0.04478399952252706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,3,0.016677333662907284
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,7,0.016623999923467636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,15,0.044480000933011375
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,7,0.04474133253097534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,15,0.016682667036851246
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,31,0.01666133354107539
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,63,0.016496000190575916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,63,0.0448586642742157
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,31,0.044639999667803444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,127,0.044677332043647766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,127,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,255,0.053802669048309326
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,511,0.03989866624275843
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,511,0.07275733351707458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,255,0.025514667232831318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,1023,0.06259733438491821
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,2047,0.20732800165812174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,1023,0.11528533697128296
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,2047,0.10904533664385478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,float16,4095,0.37797868251800537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,1,0.0260959987839063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,1,0.011770666887362799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,3,0.02608533451954524
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,8,4,128,1,float16,fp8,4095,0.200490673383077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,3,0.011653333902359009
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,7,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,7,0.026288000245889027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,31,0.010661333799362183
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,15,0.026159999271233875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,15,0.01163200040658315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,31,0.026021334032217663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,63,0.025888000925381977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,63,0.010496000448862711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,127,0.012298667182525
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,127,0.02645866572856903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,255,0.051029334465662636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,255,0.022074667116006214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,511,0.06611733138561249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,1023,0.08444266517957051
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,511,0.029898665845394135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,1023,0.04354133208592733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,1,0.04472533365090688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,fp8,2047,0.06735999882221222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,3,0.045365333557128906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,1,128,1,float16,float16,2047,0.13148799538612366
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,1,0.016730666160583496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,3,0.016544000556071598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,7,0.044981335600217186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,15,0.04514666895071665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,7,0.016650666793187458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,15,0.016629333297411602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,31,0.044922664761543274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,63,0.016528000434239704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,31,0.016789333273967106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,63,0.044351999958356224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,127,0.04527466495831808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,255,0.0545066644748052
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,127,0.019258666783571243
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,511,0.07323200007279713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,255,0.027162666122118633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,511,0.040709334115187325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,1023,0.06364800035953522
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,1023,0.11705066760381062
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,float16,2047,0.20800532897313437
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,2,128,1,float16,fp8,2047,0.10972799857457478
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,1,0.08141866823037465
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,3,0.08142933249473572
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,3,0.030202666918436687
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,7,0.030224000414212544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,1,0.02997333308060964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,7,0.08153066535790761
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,15,0.03012266755104065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,15,0.08148266871770223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,31,0.03002133220434189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,63,0.08170133332411449
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,127,0.03693866729736328
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,31,0.08127466837565105
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,255,0.09539733330408733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,63,0.030005333324273426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,127,0.0828000009059906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,255,0.05125333368778229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,511,0.0741066684325536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,511,0.1301866670449575
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,1023,0.11987200379371643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,1023,0.2104853391647339
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,1,0.00855466661353906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,1,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,3,0.008416000132759413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,float16,2047,0.38714667161305744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,8,4,128,1,float16,fp8,2047,0.21292799711227417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,3,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,15,0.008623999853928884
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,7,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,15,0.009797333429257074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,7,0.00933333362142245
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,31,0.010538666198650995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,31,0.010357333347201347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,63,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,63,0.012346666306257248
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,127,0.009599999835093817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,255,0.010629333555698395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,255,0.012330666184425354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,1023,0.013338666409254074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,511,0.016879999389251072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,2047,0.01708799973130226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,511,0.01669866715868314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,1023,0.008869333192706108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,4095,0.018911999960740406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,4095,0.010922666639089584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,2047,0.010608000059922537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,8191,0.014805333067973455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,8191,0.027621333797772724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,16383,0.01785600061217944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,16383,0.03364266703526179
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,32767,0.054229333996772766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,32767,0.02677333354949951
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,65535,0.08185066779454549
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,65535,0.052895997961362205
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,1,0.009530666594703993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,float16,131071,0.13730667034784952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,1,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,3,0.008570666735370954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,1,128,1,float16,fp8,131071,0.07135466734568278
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,3,0.009850666547815004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,7,0.008394666636983553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,7,0.010677333921194077
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,15,0.009839999799927076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,15,0.00871999996403853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,31,0.009514666472872099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,31,0.010474666953086853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,63,0.00966933307548364
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,63,0.012389333297808966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,127,0.012245333443085352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,255,0.01228800043463707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,255,0.010591999938090643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,127,0.010437333335479101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,511,0.013034666577974955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,511,0.007653333246707916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,2047,0.02203733225663503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,1023,0.008400000010927519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,1023,0.013455999394257864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,2047,0.010410666465759277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,4095,0.025429333249727886
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,4095,0.011829332758982977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,8191,0.020661332954963047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,8191,0.037178667883078255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,16383,0.0562720000743866
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,16383,0.02610666553179423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,32767,0.08351999521255493
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,32767,0.04702933132648468
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,65535,0.13851199547449747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,65535,0.07580266892910004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,1,0.008330666770537695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,1,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,float16,131071,0.23477333784103394
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,2,128,1,float16,fp8,131071,0.13642666737238565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,3,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,3,0.00980266680320104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,7,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,7,0.009706666693091393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,15,0.008778666456540426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,15,0.009877333417534828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,31,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,63,0.012335999558369318
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,31,0.01139733319481214
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,63,0.009786666681369146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,127,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,127,0.012293333808581034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,255,0.012826666235923767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,255,0.008176000167926153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,511,0.01320533330241839
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,511,0.009002666920423508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,1023,0.009109333157539368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,2047,0.015205333630243937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,2047,0.02867199977238973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,1023,0.01939733326435089
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,4095,0.041050667564074196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,4095,0.018186666071414948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,16383,0.09130133191744487
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,8191,0.029045333464940388
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,8191,0.05963733295599619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,16383,0.047584002216657005
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,32767,0.1306986709435781
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,32767,0.07674133280913036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,65535,0.22088533639907837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,65535,0.1306719978650411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,1,0.0462719996770223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,float16,131071,0.405290683110555
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,1,0.015194666882356008
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,3,0.04610666632652283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,8,4,128,1,float16,fp8,131071,0.22185067335764566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,3,0.01505600040157636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,7,0.04615999758243561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,15,0.04595733185609182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,31,0.04595200220743815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,15,0.015087999403476715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,63,0.04577599962552389
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,31,0.015109332899252573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,63,0.014949332922697067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,127,0.047423998514811196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,127,0.017978666971127193
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,255,0.026261332134405773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,255,0.055760001142819725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,511,0.07569600145022075
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,1,0.08166933556397755
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,1,0.03010133405526479
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,511,0.04035199930270513
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,float16,1023,0.11994666854540507
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,1,128,1,float16,fp8,1023,0.06382933259010315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,3,0.03001066545645396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,3,0.0815413345893224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,7,0.08143466711044312
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,7,0.02997333308060964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,15,0.03017599880695343
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,31,0.08131200075149536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,15,0.0819946676492691
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,31,0.03014400104681651
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,63,0.08183466891447704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,63,0.03014933317899704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,127,0.03885333240032196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,127,0.08404800295829773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,255,0.09647466739018758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,255,0.05271466573079427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,1023,0.21232000986735025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,1,0.15079466501871744
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,511,0.07584000130494435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,1,0.06357866525650024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,float16,511,0.13226133584976196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,2,128,1,float16,fp8,1023,0.12181867162386577
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,3,0.15070399641990662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,3,0.06297599772612254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,7,0.15090133746465048
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,15,0.15008533000946045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,7,0.06390400230884552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,15,0.0634933312733968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,31,0.15319466590881348
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,31,0.06331733365853627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,63,0.15372799833615622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,63,0.06237333516279856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,127,0.153738667567571
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,255,0.09488532940546672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,127,0.07369066774845123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,255,0.17826666434605917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,511,0.14064000050226846
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,511,0.2455093264579773
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,float16,1023,0.3956586519877116
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,8,4,128,1,float16,fp8,1023,0.2323039968808492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,1,0.08342933654785156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,1,0.0262719988822937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,3,0.026133333643277485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,3,0.0835093359152476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,7,0.08343467116355896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,15,0.08318933347860973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,15,0.026762666801611584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,7,0.0266239990790685
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,31,0.025994665920734406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,31,0.08338666955629985
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,63,0.025781333446502686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,63,0.08492799599965413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,127,0.08628267049789429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,127,0.037674665451049805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,255,0.04906666775544485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,255,0.09959466258684795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,1,0.06685866912206014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,1,0.15085333585739136
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,3,0.15105600158373514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,3,0.06691200037797292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,fp8,511,0.07027733325958252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,15,0.15134400129318237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,7,0.06644266843795776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,1,128,1,float16,float16,511,0.13666133085886636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,15,0.06672533353169759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,7,0.15197333693504333
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,31,0.15481066703796387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,31,0.06663999954859416
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,63,0.06694933275381725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,63,0.15587733189264932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,127,0.1562986671924591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,255,0.09820266564687093
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,127,0.07595199843247731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,1,0.29102933406829834
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,255,0.18173333009084067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,1,0.12476266423861186
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,float16,511,0.24918399254480997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,2,128,1,float16,fp8,511,0.143994669119517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,3,0.291647990544637
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,7,0.291098674138387
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,3,0.12495999534924825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,15,0.29517332712809247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,7,0.12477866808573405
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,15,0.12532266974449158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,31,0.299344003200531
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,31,0.1251626710096995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,63,0.2980853319168091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,127,0.29740800460179645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,63,0.12559466560681662
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,127,0.1394613285859426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,1,0.008597333605090777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,255,0.3447999954223633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,255,0.18250133593877158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,1,0.0103946669648091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,3,0.009648000200589498
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,float16,511,0.47616533438364667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,3,0.008703999842206636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,7,0.00860799973209699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,7,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,8,4,128,1,float16,fp8,511,0.2725013295809428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,15,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,15,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,31,0.009472000102202097
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,63,0.009488000224033991
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,31,0.010415999839703241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,63,0.012282667060693106
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,127,0.009759999811649323
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,127,0.012362666428089142
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,511,0.012693333129088083
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,255,0.01240533341964086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,511,0.008053333188096682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,1023,0.013343999783198038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,2047,0.022490667800108593
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,1023,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,4095,0.026074667771657307
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,2047,0.010911999891201654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,4095,0.012901333471139273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,8191,0.037952000896135964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,8191,0.01775466650724411
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,16383,0.05598933498064677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,16383,0.029109333952267964
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,32767,0.08477866649627686
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,32767,0.04295999805132548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,65535,0.13638400038083395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,65535,0.07368533313274384
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,1,0.008576000109314919
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,1,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,float16,131071,0.23305066426595053
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,1,128,1,float16,fp8,131071,0.11784000198046367
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,3,0.008592000231146812
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,3,0.009808000177145004
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,7,0.008736000085870424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,15,0.008837333569924036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,7,0.00956266683836778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,31,0.009445333232482275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,15,0.009935999910036722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,31,0.010634666929642359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,63,0.009674666449427605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,127,0.009839999799927076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,63,0.01231466606259346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,127,0.01231466606259346
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,255,0.012432000289360682
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,1023,0.019109333554903667
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,255,0.007370666911204656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,511,0.01302933320403099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,511,0.008080000057816505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,1023,0.009392000113924345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,2047,0.029205332199732464
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,4095,0.042064001162846885
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,2047,0.015189333508412043
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,4095,0.01815466706951459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,8191,0.06007466713587443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,8191,0.0290133332212766
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,16383,0.08933333555857341
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,16383,0.0476746658484141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,32767,0.12995733817418417
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,32767,0.07733866572380066
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,65535,0.1307360033194224
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,65535,0.21942400932312012
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,1,0.0106133334338665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,1,0.006122666721542676
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,3,0.010522666076819101
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,fp8,131071,0.22087466716766357
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,2,128,1,float16,float16,131071,0.40651198228200275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,3,0.0063573333124319715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,7,0.010458666831254959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,7,0.006309333567818006
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,15,0.010170666500926018
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,15,0.006170666466156642
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,31,0.010506667196750641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,127,0.006688000013430913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,63,0.010362666721145311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,31,0.005978666866819064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,63,0.005968000118931134
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,127,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,255,0.013749333719412485
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,511,0.019381333142518997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,255,0.0075626665105422335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,511,0.008698666468262672
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,1023,0.015461333096027374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,1023,0.028005334238211315
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,2047,0.0433599998553594
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,2047,0.02089600016673406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,4095,0.06856533388296764
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,4095,0.03242133309443792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,8191,0.08669333656628926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,8191,0.046341334780057274
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,16383,0.06924266616503398
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,16383,0.12772267063458762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,32767,0.21152534087498984
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,32767,0.11593600114186604
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,65535,0.20786132415135702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,1,0.008538666491707167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,65535,0.3824479977289836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,3,0.008714666590094566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,1,0.009888000165422758
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,float16,131071,0.7222239971160889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,8,4,128,1,float16,fp8,131071,0.3938666582107544
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,3,0.009861333295702934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,15,0.008943999807039896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,7,0.009749333063761393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,7,0.008538666491707167
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,15,0.0099093330403169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,31,0.009642666826645533
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,31,0.010645333677530289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,63,0.012485332787036896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,63,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,127,0.012421333541472753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,127,0.009701333319147428
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,255,0.012517333030700684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,255,0.007376000285148621
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,511,0.00821333316465219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,511,0.01293333371480306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,1023,0.01926400015751521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,1023,0.009402666861812273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,2047,0.029135999580224354
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,2047,0.015040000279744467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,4095,0.01825599993268649
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,4095,0.04173333446184794
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,8191,0.06099733213583628
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,8191,0.02932800104220708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,16383,0.09123200178146362
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,16383,0.04771733283996582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,32767,0.13127467036247253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,32767,0.0777759999036789
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,65535,0.21995200713475546
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,1,0.010410666465759277
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,65535,0.130730668703715
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,1,0.006031999985376994
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,3,0.010234666367371878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,float16,131071,0.40585601329803467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,3,0.006271999950210254
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,15,0.0063040001938740415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,7,0.006405333057045937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,7,0.010373333469033241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,15,0.010154666379094124
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,1,128,1,float16,fp8,131071,0.2221226692199707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,63,0.010399999717871347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,31,0.010250666489203772
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,31,0.006362666686375936
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,127,0.006448000048597653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,127,0.010405333091815313
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,63,0.0061493335912625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,255,0.013295999417702356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,255,0.008154666672150293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,511,0.019440000255902607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,511,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,1023,0.02811199923356374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,1023,0.015285332997639975
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,2047,0.04397333165009817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,4095,0.06805866460005443
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,2047,0.02060266708334287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,4095,0.03260799994071325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,8191,0.08764266967773438
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,8191,0.04626133541266123
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,16383,0.12852799892425537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,16383,0.06942933301130931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,32767,0.21157866716384888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,32767,0.11664000153541565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,1,0.008639999975760778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,1,0.009296000003814697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,65535,0.3805333375930786
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,65535,0.20975999037424722
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,3,0.008176000167926153
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,7,0.008112000301480293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,3,0.009248000259200731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,float16,131071,0.7200799783070883
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,32,1,4,2,128,1,float16,fp8,131071,0.3933759927749634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,7,0.009248000259200731
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,15,0.008442666381597519
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,31,0.009109333157539368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,15,0.009349333122372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,31,0.009989333028594652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,63,0.009349333122372627
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,127,0.00921066664159298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,127,0.011994666109482447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,63,0.011973333855470022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,255,0.010186666622757912
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,255,0.01192533348997434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,511,0.016336000214020412
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,511,0.015941333025693893
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,1023,0.017429333180189133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,1023,0.017322666943073273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,2047,0.01743999992807706
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,2047,0.01728533332546552
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,4095,0.017887999614079792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,4095,0.01754666616519292
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,8191,0.01811733345190684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,8191,0.012479999413092932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,16383,0.019834666202465694
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,16383,0.014917333920796713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,32767,0.02441066751877467
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,32767,0.019023999571800232
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,65535,0.029498666524887085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,65535,0.023984000086784363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,1,0.008047999814152718
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,1,0.009066666786869368
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,float16,131071,0.039850667119026184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,1,128,1,float16,fp8,131071,0.03177600105603536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,3,0.00916800027092298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,3,0.008063999935984612
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,7,0.00810666692753633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,7,0.009178666397929192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,15,0.00855466661353906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,15,0.009290666629870733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,31,0.00916800027092298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,31,0.009957333405812582
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,63,0.012106666962305704
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,63,0.009173333023985228
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,127,0.009136000027259191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,127,0.011898666620254517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,255,0.01028266673286756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,255,0.011882666498422623
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,511,0.016208000481128693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,1023,0.017445333302021027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,511,0.016010666886965435
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,1023,0.017237332959969837
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,2047,0.01736533393462499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,4095,0.015599999576807022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,4095,0.010314666976531347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,2047,0.017136000096797943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,8191,0.01249066616098086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,8191,0.01937599976857503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,16383,0.020687999824682873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,16383,0.015018666783968607
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,32767,0.025311999022960663
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,32767,0.01903466631968816
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,65535,0.024165332317352295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,65535,0.031583999594052635
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,1,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,1,0.009152000149091085
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,float16,131071,0.06039999922116598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,3,0.008165333420038223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1,1,4,2,128,1,float16,fp8,131071,0.03502399971087774
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,3,0.009093333035707474
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,7,0.008266666904091835
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,7,0.009253333633144697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,15,0.008570666735370954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,31,0.009183999771873156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,15,0.009461333354314169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,31,0.009973333527644476
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,63,0.00921066664159298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,63,0.011952000359694162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,127,0.009242666885256767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,255,0.010527999450763067
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,127,0.01198400060335795
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,255,0.012144000579913458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,511,0.01623999948302905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,511,0.016506666938463848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,1023,0.017610666652520496
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,1023,0.017210666090250015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,2047,0.017845333864291508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,4095,0.016352000335852306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,2047,0.017322666943073273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,4095,0.010389333590865135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,8191,0.01926400015751521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,8191,0.012527999778588613
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,16383,0.02102400114138921
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,16383,0.015024000157912573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,32767,0.01922133316596349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,32767,0.025519999365011852
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,65535,0.03188266605138779
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,65535,0.024106666445732117
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,1,0.00810666692753633
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,float16,131071,0.06172266602516174
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,1,0.009162666896979014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,3,0.008298666526873907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,1,128,1,float16,fp8,131071,0.035162667433420815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,3,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,7,0.008117333054542542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,7,0.009285333255926767
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,15,0.00855466661353906
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,15,0.009397333487868309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,31,0.010266666611035665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,31,0.009125333279371262
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,63,0.009194666519761086
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,127,0.00926399976015091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,63,0.011936000237862269
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,127,0.012159999459981918
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,255,0.010357333347201347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,255,0.01211200033624967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,511,0.016442666451136272
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,511,0.016176000237464905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,1023,0.017456000049908955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,1023,0.01724799970785777
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,2047,0.015461333096027374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,4095,0.01657066618402799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,2047,0.009189333145817121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,4095,0.010490667074918747
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,8191,0.0204373337328434
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,8191,0.01267733300725619
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,16383,0.022282667458057404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,32767,0.03626133253177007
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,16383,0.015098666151364645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,32767,0.02093333254257838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,65535,0.0507893313964208
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,65535,0.02681066592534383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,1,0.010239999741315842
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,1,0.006159999718268712
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,3,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,float16,131071,0.08390399813652039
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2,1,4,2,128,1,float16,fp8,131071,0.055904000997543335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,3,0.006229333579540253
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,7,0.010314666976531347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,7,0.006341333190600078
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,15,0.010288000106811523
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,15,0.006330666442712148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,31,0.01055466632048289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,31,0.006224000205596288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,63,0.0103946669648091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,63,0.006330666442712148
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,127,0.010480000327030817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,255,0.013248000293970108
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,127,0.006629333520929019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,255,0.007797333101431529
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,511,0.01956266661485036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,511,0.008863999818762144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,1023,0.02808533360560735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,1023,0.015354666858911514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,2047,0.04368533194065094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,2047,0.020960000654061634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,4095,0.0683786670366923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,4095,0.03358400116364161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,8191,0.047050664822260536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,8191,0.08974400162696838
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,16383,0.12980266412099203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,16383,0.069882666071256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,32767,0.21304533878962198
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,32767,0.1179253359635671
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,1,0.010757333288590113
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,1,0.006074666976928711
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,3,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,3,0.006128000095486641
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,7,0.010421333213647207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,fp8,65535,0.21173334121704102
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,7,0.007093333328763644
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,1,128,1,float16,float16,65535,0.38281599680582684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,15,0.010480000327030817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,15,0.0063733334342638654
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,31,0.010645333677530289
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,31,0.00625599982837836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,63,0.010362666721145311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,63,0.006090666477878888
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,127,0.006671999891599019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,255,0.020970667401949566
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,127,0.010725333044926325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,511,0.02824000020821889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,255,0.008277333031098047
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,511,0.014208000153303146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,1023,0.046154667933781944
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,1023,0.024304000039895374
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,2047,0.06639466683069865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,2047,0.03067733347415924
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,4095,0.08628799517949422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,4095,0.044906665881474815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,8191,0.1264639993508657
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,8191,0.0681333343187968
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,16383,0.21211733420689902
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,16383,0.11407466729482015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,32767,0.38005868593851727
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,1,0.008165333420038223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,32767,0.20521599054336548
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,3,0.008330666770537695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,1,0.009402666861812273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,3,0.009290666629870733
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,7,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,float16,65535,0.7179360389709473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,64,1,4,2,128,1,float16,fp8,65535,0.3877973159154256
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,7,0.009482666850090027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,15,0.008496000121037165
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,31,0.009141333401203156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,15,0.009477333476146063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,31,0.010202666744589806
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,63,0.012138667205969492
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,255,0.01032533310353756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,63,0.009354666496316591
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,127,0.012005332857370377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,127,0.009530666594703993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,255,0.012063999970753988
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,511,0.01648533344268799
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,1023,0.01764800027012825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,511,0.016362667083740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,2047,0.017407999684413273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,1023,0.01727466657757759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,2047,0.009205333267649015
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,4095,0.017743999759356182
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,8191,0.02163733293612798
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,4095,0.010538666198650995
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,8191,0.012810666114091873
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,16383,0.015119999647140503
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,16383,0.023823998868465424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,32767,0.0372533326347669
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,32767,0.021344001094500225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,65535,0.05340800185998281
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,65535,0.026752000053723652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,1,0.008293333152929941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,1,0.009450666606426239
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,3,0.00816000004609426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,float16,131071,0.08568533261617024
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,1,128,1,float16,fp8,131071,0.056474665800730385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,3,0.009477333476146063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,7,0.008256000156203905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,7,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,15,0.008522666369875273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,15,0.009402666861812273
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,31,0.009162666896979014
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,31,0.010373333469033241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,63,0.009423999736706415
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,63,0.01209066684047381
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,127,0.009455999980370203
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,127,0.012080000092585882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,255,0.010469333579142889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,511,0.016362667083740234
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,255,0.012165332833925882
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,511,0.01681600014368693
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,1023,0.008256000156203905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,1023,0.013424000392357508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,2047,0.017749333133300144
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,4095,0.018543999642133713
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,2047,0.009119999905427298
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,4095,0.010469333579142889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,8191,0.027482666075229645
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,8191,0.014186666657527288
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,16383,0.03372266640265783
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,16383,0.016895999511082966
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,32767,0.05389333268006643
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,32767,0.03206400076548258
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,65535,0.08157866696516673
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,65535,0.044400001565615334
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,float16,131071,0.13756799697875977
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,1,0.010687999427318573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,1,0.006325333068768184
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,3,0.01071999967098236
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,4,1,4,2,128,1,float16,fp8,131071,0.07706666489442189
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,3,0.00625599982837836
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,7,0.010874666273593903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,15,0.010672000547250112
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,7,0.0063146669417619705
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,15,0.006346666564544042
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,31,0.01073066641887029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,63,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,31,0.006010666489601135
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,63,0.006842666616042455
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,127,0.0069440001000960665
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,127,0.010832000523805618
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,255,0.020554666717847187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,255,0.008570666735370954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,511,0.028197333216667175
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,511,0.01441066712141037
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,1023,0.02446399877468745
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,1023,0.04588800172011057
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,2047,0.06676266590754192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,2047,0.03225066761175791
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,4095,0.08858666817347209
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,4095,0.04597333570321401
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,8191,0.06837333242098491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,8191,0.1277653376261393
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,1,0.015317333241303762
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,16383,0.214303990205129
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,16383,0.11529599626859029
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,1,0.008447999755541483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,3,0.015706667055686314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,3,0.006650666395823161
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,float16,32767,0.3848746617635091
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,7,0.015557333827018738
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,15,0.015381333728631338
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,15,0.006544000158707301
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,1,128,1,float16,fp8,32767,0.2074133356412252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,31,0.015509333461523056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,7,0.007770666852593422
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,31,0.006762666627764702
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,63,0.015205333630243937
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,63,0.0068853336075941724
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,127,0.01578666642308235
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,127,0.007290666922926903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,255,0.03030933439731598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,255,0.013631999492645264
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,511,0.046810666720072426
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,511,0.023845332364241283
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,1023,0.06226666768391927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,1023,0.030213333666324615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,2047,0.08598933617273967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,2047,0.04452266792456309
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,4095,0.06758933266003926
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,4095,0.12738133470217386
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,8191,0.11363733808199565
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,8191,0.21318399906158447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,16383,0.3833386500676473
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,1,0.00697066696981589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,3,0.015509333461523056
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,1,0.015610666324694952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,16383,0.20590933163960776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,3,0.006704000135262807
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,float16,32767,0.7237866719563802
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,7,0.015546667079130808
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,7,0.007461333026488622
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,15,0.00679466687142849
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,15,0.015749332805474598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,128,1,4,2,128,1,float16,fp8,32767,0.38738131523132324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,31,0.015599999576807022
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,31,0.006613333399097125
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,63,0.015311999867359797
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,127,0.015344000111023584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,63,0.009695999945203463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,127,0.008293333152929941
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,255,0.013797332843144735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,255,0.02998399982849757
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,511,0.0467199981212616
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,511,0.02421333392461141
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,1023,0.06202666461467743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,2047,0.045194665590922035
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,1023,0.03202133377393087
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,4095,0.1276800036430359
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,4095,0.06805333495140076
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,2047,0.08841600020726521
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,float16,8191,0.2161066730817159
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,1,0.025445332129796345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,3,0.025290665527184803
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,3,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,1,128,1,float16,fp8,8191,0.11493333180745442
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,1,0.010458666831254959
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,7,0.025279998779296875
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,7,0.009765333185593287
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,15,0.025221332907676697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,15,0.009743999689817429
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,31,0.025487999121348064
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,63,0.02499199906984965
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,31,0.011231999844312668
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,63,0.01032533310353756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,127,0.02552533398071925
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,255,0.04966933528582255
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,255,0.023578666150569916
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,511,0.06481066842873891
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,1023,0.04481600224971771
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,511,0.030767999589443207
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,2047,0.06749866902828217
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,2047,0.12945066889127096
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,1023,0.08329066634178162
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,4095,0.21794666846593222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,4095,0.11389866471290588
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,1,0.025653332471847534
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,1,0.009829333052039146
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,3,0.025477332373460133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,3,0.00972800018886725
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,float16,8191,0.3892693519592285
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,7,0.025642665723959606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,7,0.009546666716535887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,15,0.025349333882331848
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,31,0.025994665920734406
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,256,1,4,2,128,1,float16,fp8,8191,0.20572266976038614
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,15,0.010442666709423065
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,31,0.009946666657924652
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,63,0.02536533276240031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,63,0.00979200005531311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,127,0.02550933261712392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,511,0.06519466638565063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,255,0.02399466683467229
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,255,0.04987200101216634
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,511,0.03173866619666418
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,1023,0.0839359958966573
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,1023,0.04558933277924856
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,2047,0.13076800107955933
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,1,0.04445866743723551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,1,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,3,0.04461333155632019
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,2047,0.06881600121657054
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,3,0.01669866715868314
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,7,0.04451199869314829
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,7,0.016549333930015564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,15,0.04466133316357931
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,15,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,31,0.04534933467706045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,float16,4095,0.21957866350809732
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,1,128,1,float16,fp8,4095,0.11520533760388692
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,31,0.016650666793187458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,63,0.044293334086736046
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,63,0.016602666427691776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,127,0.04554666578769684
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,127,0.019066666563351948
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,255,0.05423999826113383
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,511,0.0399893323580424
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,255,0.025594666600227356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,511,0.07262933254241943
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,1023,0.11533866326014201
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,1023,0.06266666452089946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,2047,0.20721600453058878
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,2047,0.10884799559911092
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,1,0.008346666892369589
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,1,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,3,0.00847999999920527
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,3,0.009482666850090027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,7,0.008485333373149237
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,7,0.00956266683836778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,15,0.00867733359336853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,15,0.009690666571259499
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,float16,4095,0.392959992090861
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,31,0.009189333145817121
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,31,0.010314666976531347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,63,0.009632000078757605
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,63,0.012202666451533636
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,127,0.00938666673998038
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,512,1,4,2,128,1,float16,fp8,4095,0.20088533560434976
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,127,0.012122667084137598
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,255,0.01051733394463857
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,511,0.016821333517630894
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,1023,0.01357866699496905
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,511,0.01659199967980385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,1023,0.008240000034372011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,2047,0.017312000195185345
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,4095,0.018090666582187016
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,2047,0.00915733352303505
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,4095,0.010640000303586325
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,8191,0.014192000031471252
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,16383,0.0332640012105306
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,8191,0.028175999720891316
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,16383,0.01714666684468587
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,32767,0.05436266462008158
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,32767,0.03253866732120514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,65535,0.08202666540940602
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,65535,0.04499199986457825
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,1,0.008357333640257517
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,float16,131071,0.13640000422795615
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,1,128,1,float16,fp8,131071,0.07734933495521545
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,1,0.009775999933481216
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,3,0.008383999889095625
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,3,0.009589333087205887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,7,0.008416000132759413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,7,0.009519999846816063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,15,0.00871999996403853
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,15,0.009861333295702934
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,31,0.009578666960199675
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,31,0.01028266673286756
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,63,0.009338666374484697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,63,0.01232533281048139
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,127,0.009466666728258133
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,127,0.012213333199421564
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,255,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,255,0.012357333054145178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,511,0.012879999975363413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,1023,0.008197333042820295
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,511,0.008021333565314611
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,2047,0.022133332987626392
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,1023,0.013525333255529404
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,2047,0.010341333225369453
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,4095,0.025631998976071674
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,8191,0.036570665736993156
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,4095,0.011802667131026586
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,8191,0.02081599955757459
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,16383,0.05599466462930044
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,16383,0.026181332767009735
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,32767,0.08347200353940327
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,32767,0.04615999758243561
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,65535,0.13565867145856222
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,65535,0.07530133426189423
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,1,0.044639999667803444
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,float16,131071,0.2342133323351542
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,8,1,4,2,128,1,float16,fp8,131071,0.13617600003878275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,1,0.016517333686351776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,3,0.04494399825731913
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,3,0.016490666816631954
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,7,0.04541866481304169
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,7,0.01659199967980385
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,15,0.044853334625562034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,15,0.016693333784739178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,31,0.04488533238569895
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,31,0.016517333686351776
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,63,0.04445866743723551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,63,0.016399999459584553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,127,0.04574933151404063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,127,0.01937066639463107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,255,0.054245332876841225
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,255,0.026895999908447266
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,511,0.07398933172225952
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,511,0.04102399945259094
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,1023,0.11626133322715759
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,1023,0.0637600024541219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,1,0.0813920001188914
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,1,0.030207999050617218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,3,0.08152533570925395
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,7,0.08165333171685536
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,3,0.030293333033720653
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,float16,2047,0.20823466777801514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,7,0.030106666187445324
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,1,128,1,float16,fp8,2047,0.11042666435241699
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,15,0.08115733166535695
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,15,0.03001066545645396
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,31,0.08136000235875447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,31,0.030058667063713074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,63,0.08107199768225352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,127,0.08343999584515889
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,63,0.029989334444204967
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,255,0.09662933150927226
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,127,0.03701333453257879
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,255,0.05114666620890299
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,511,0.13085333506266275
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,511,0.07457066575686137
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,1023,0.2104746699333191
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,1,0.030181333422660828
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,1023,0.12062933047612508
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,1,0.08155199885368347
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,3,0.030373332401116688
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,float16,2047,0.38870398203531903
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,3,0.0817333310842514
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,7,0.08186133205890656
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,7,0.029978667696317036
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,15,0.030245333909988403
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,15,0.08147199948628743
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,1024,1,4,2,128,1,float16,fp8,2047,0.21241599321365356
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,63,0.08241599798202515
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,63,0.02991466720898946
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,31,0.08180800080299377
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,31,0.030069333811601002
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,127,0.08441600203514099
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,255,0.05249600112438202
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,127,0.03937600056330363
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,255,0.09667199850082397
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,511,0.1328426698843638
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,1,0.1509066621462504
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,1,0.06457599997520447
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,3,0.0637600024541219
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,7,0.15104533235232034
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,3,0.15060800313949585
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,511,0.07588799794514973
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,15,0.15025066335995993
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,15,0.06252266466617584
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,fp8,1023,0.12210133671760559
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,7,0.06468266745408376
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,31,0.1524799962838491
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,31,0.06345599889755249
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,63,0.1540000041325887
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,63,0.0629066675901413
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,127,0.15570666392644247
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,1,128,1,float16,float16,1023,0.2157920002937317
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,127,0.07454933226108551
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,1,0.00949866697192192
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,255,0.17975467443466187
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,511,0.24687999486923218
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,511,0.14129599928855896
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,255,0.09553066889444987
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,1,0.009525333220760027
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,7,0.009594666461149851
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,3,0.009653333574533463
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,7,0.0084906667470932
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,3,0.009637333452701569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,15,0.00874133345981439
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,15,0.009717333440979322
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,31,0.010362666721145311
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,63,0.009519999846816063
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,31,0.010586666564146677
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,63,0.012421333541472753
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,float16,1023,0.3985280195871989
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,127,0.009685333197315535
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,127,0.01221866657336553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,255,0.010533332824707031
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,511,0.012719999998807907
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,255,0.01239466667175293
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,511,0.007823999971151352
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,1023,0.013306666165590286
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,2048,1,4,2,128,1,float16,fp8,1023,0.23304533958435059
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,2047,0.010415999839703241
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,1023,0.008394666636983553
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,2047,0.021967999637126923
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,4095,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,4095,0.026250667870044708
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,8191,0.03749866783618927
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,16383,0.056287998954455055
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,8191,0.020986666282018025
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,16383,0.0262773334980011
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,32767,0.08501866459846497
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,32767,0.04678399860858917
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,65535,0.07569066683451335
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,1,0.008320000022649765
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,1,0.009797333429257074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,65535,0.13657066226005554
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,float16,131071,0.23056532939275107
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,1,128,1,float16,fp8,131071,0.13633599877357483
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,3,0.008634666601816813
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,3,0.009679999823371569
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,7,0.008613333106040955
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,7,0.009557333464423815
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,15,0.008816000074148178
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,15,0.009797333429257074
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,31,0.00960533320903778
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,31,0.010559999694426855
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,63,0.009599999835093817
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,63,0.012256000190973282
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,127,0.009658666948477427
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,255,0.012586666891972223
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,127,0.012229333321253458
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,511,0.007861333588759104
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,511,0.013253333667914072
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,255,0.007311999797821045
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,2047,0.02888533224662145
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,1023,0.00943999985853831
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,1023,0.01913600042462349
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,4095,0.04196799794832865
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,4095,0.017877332866191864
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,8191,0.06002133091290792
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,8191,0.028853334486484528
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,16383,0.08995733658472697
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,16383,0.047226667404174805
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,32767,0.13040000200271606
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,32767,0.07714666426181793
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,65535,0.12986133495966592
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,65535,0.21806933482487997
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,float16,131071,0.4049866596857707
TRTLLM,1.0.0rc3,NVIDIA H100 80GB HBM3,generation_attention,torch_flow,16,1,4,2,128,1,float16,fp8,131071,0.22028799851735434
