framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1,0.025706666211287182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,3,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,7,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,15,0.025754667818546295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,15,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,31,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,31,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,63,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,63,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,127,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,3,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,127,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,255,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,255,0.025424001117547352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1023,0.0591786652803421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1023,0.0524479995171229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,511,0.039733332892258964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,2047,0.09943999846776326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,2047,0.08699733018875122
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,4095,0.1737226645151774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,7,0.02573866645495097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,4095,0.15038933356602988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,3,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,511,0.034186666210492454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,511,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1023,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,2047,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,4095,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,4095,0.020831999679406483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,8191,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,8191,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,16383,0.0359253336985906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,16383,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,63,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,255,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,511,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1023,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,2047,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,2047,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,4095,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,4095,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,8191,0.03572266548871994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,8191,0.029680001238981884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,16383,0.05801600217819214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,16383,0.051813334226608276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1,0.03956266740957896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,3,0.041482667128245033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,3,0.03979199876387914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,7,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,7,0.03990400085846583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,15,0.0415786678592364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,15,0.039520000418027244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,31,0.04192533095677694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,31,0.03959999978542328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,63,0.04185600082079569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,63,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,127,0.042549331982930504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,127,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,255,0.04363200068473816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,255,0.039488000174363456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,511,0.06478400031725566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,511,0.05858133236567179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1023,0.1011893351872762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1023,0.0888853371143341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,2047,0.17640533049901327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,2047,0.15440000096956888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,3,0.01250133290886879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,3,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,7,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,7,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,15,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,15,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,31,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,31,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,63,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,63,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,127,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,127,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,255,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1023,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1023,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,2047,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,2047,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,4095,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,4095,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,8191,0.062224000692367554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,8191,0.05403199791908264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,16383,0.10498666763305664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,16383,0.09057600299517314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1,0.07446399827798207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1,0.06835199892520905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,3,0.07430399954319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,3,0.0682666649421056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,7,0.07520533104737599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,7,0.06829866766929626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,15,0.06845866640408833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,15,0.07558933397134145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,31,0.07437866429487865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,31,0.06849066913127899
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,63,0.07423999905586243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,63,0.06681600213050842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,127,0.07460799813270569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,127,0.06676266590754192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,255,0.07468800246715546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,255,0.06871466835339864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,511,0.11547199885050456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,511,0.10336533188819885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1023,0.1874613364537557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1023,0.1642453372478485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,1,0.13804800311724344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,1,0.125791996717453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,3,0.1378613313039144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,3,0.12588266531626383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,7,0.13844266533851624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,7,0.12678933143615723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,15,0.13802133003870645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,15,0.1257866621017456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,31,0.13804800311724344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,31,0.12590932846069336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,63,0.1381013294061025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,63,0.1258026659488678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,127,0.13819733262062073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,127,0.1278986632823944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,255,0.13576533397038779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,255,0.12391466895739238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,1,0.2637173334757487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,1,0.24356800317764282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,3,0.2651253342628479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,3,0.24395199616750082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,7,0.26377065976460773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,7,0.24276266495386759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,15,0.26337067286173504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,15,0.24258132775624594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,31,0.26307199398676556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,31,0.24260266621907553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,63,0.2638346751530965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,63,0.24246400594711304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,127,0.26311467091242474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,127,0.2423893411954244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,3,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,15,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,15,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,31,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,31,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,63,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,63,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,127,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,127,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,255,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,255,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,511,0.018101333330074947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,511,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1023,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1023,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,2047,0.04013866682847341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,2047,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,4095,0.06158933540185293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,4095,0.05481599768002828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,8191,0.10506666700045268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,8191,0.09143466750780742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,16383,0.1897546648979187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,16383,0.16290133198102316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,1,0.5147200028101603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,1,0.47389864921569824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,3,0.5150186618169149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,3,0.4743093252182007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,7,0.5149279832839966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,7,0.47388267517089844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,15,0.5147626797358195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,15,0.4739253520965576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,31,0.5146880149841309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,31,0.47383467356363934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,63,0.514784018198649
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,63,0.4743146498998006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,1,1.0155306657155354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,1,0.934666633605957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,3,1.015877326329549
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,3,0.9340960184733073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,7,1.0158133506774902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,7,0.9337173302968343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,15,1.0150720278422039
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,15,0.9339839617411295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,31,1.0149119695027669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,31,0.9339040120442709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,3,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,3,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,7,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,7,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,15,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,15,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,31,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,31,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,63,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,63,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,127,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,127,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,255,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,255,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,511,0.024933333198229473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,511,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1023,0.03575466573238373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1023,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,2047,0.058037335673967995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,2047,0.05193600058555603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,4095,0.09507733583450317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,4095,0.08309333523114522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,8191,0.17083734273910522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,8191,0.1465226709842682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,3,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,3,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,7,0.02942933390537898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,7,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,15,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,15,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,31,0.027669332921504974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,31,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,63,0.02940266579389572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,63,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,127,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,127,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,255,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,255,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,511,0.05392533540725708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,511,0.037834666669368744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1023,0.09040000041325887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1023,0.055919999877611794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,2047,0.1693120002746582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,2047,0.09504533807436626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,4095,0.321946660677592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,4095,0.16914133230845133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,7,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,127,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,511,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,511,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1023,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1023,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,2047,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,2047,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,4095,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,4095,0.018170667191346485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,8191,0.03452266752719879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,8191,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,16383,0.05628266433874766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,16383,0.037104000647862755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,31,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,511,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1023,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,2047,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,2047,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,4095,0.03514666606982549
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,4095,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,8191,0.05622399846712748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,8191,0.03559466699759165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,16383,0.09893332918485005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,16383,0.056133334835370384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1,0.04799466828505198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1,0.04141866664091746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,3,0.04702933132648468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,3,0.03990400085846583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,7,0.045994664231936135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,7,0.03996799886226654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,15,0.0481279989083608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,15,0.04173333446184794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,31,0.04790399968624115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,31,0.04167999823888143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,63,0.04695466657479604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,63,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,127,0.04780266682306925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,127,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,255,0.05267733335494995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,255,0.041637333730856575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,511,0.0916373332341512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,511,0.06200533111890157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1023,0.1665493349234263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1023,0.09709333380063374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,2047,0.3194613258043925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,2047,0.17082132895787558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,3,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,3,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,7,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,7,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,15,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,15,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,31,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,31,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,63,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,127,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,255,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,511,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,511,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1023,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1023,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,2047,0.037717332442601524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,2047,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,4095,0.0592853327592214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,4095,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,8191,0.10107733805974324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,8191,0.06033066908518473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,16383,0.1840426723162333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,16383,0.10099732875823975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1,0.08532266815503438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1,0.07076799869537354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,3,0.08504533767700195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,3,0.0705813318490982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,7,0.08454400300979614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,7,0.07061866422494252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,15,0.08473599950472514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,15,0.07069866855939229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,31,0.08468799789746602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,31,0.07286400099595387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,63,0.0849226713180542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,63,0.0726506660381953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,127,0.08452266454696655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,127,0.07253866891066234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,255,0.09117333094278972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,255,0.07267199953397115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,511,0.16861865917841592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,511,0.11041067043940227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1023,0.3148053288459778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1023,0.17932265996932983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,1,0.1569439967473348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,1,0.1344053347905477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,3,0.1567200024922689
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,3,0.13407466808954874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,7,0.15685866276423135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,7,0.13385599851608276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,15,0.15676800409952799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,15,0.13361066579818726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,31,0.15658666690190634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,31,0.1338933308919271
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,63,0.15652267138163248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,63,0.13409066200256348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,127,0.156442662080129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,127,0.13403200109799704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,255,0.17013333241144815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,255,0.1321440041065216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,1,0.30018667380015057
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,1,0.255018671353658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,3,0.30153600374857586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,3,0.2548533280690511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,7,0.2999093333880107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,7,0.254533330599467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,15,0.3003679911295573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,15,0.2550400098164876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,31,0.3002026677131653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,31,0.2549333373705546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,63,0.30026666323343915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,63,0.2547626694043477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,127,0.2985333402951558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,127,0.25465067227681476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,3,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,3,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,7,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,7,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,15,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,15,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,31,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,31,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,63,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,63,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,127,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,127,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,255,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,255,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,511,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,511,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1023,0.0335413341720899
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1023,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,2047,0.05900266766548157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,2047,0.03937066594759623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,4095,0.10147733489672343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,4095,0.06014933188756307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,8191,0.18421866496404013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,8191,0.10241066416104634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,16383,0.353653351465861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,16383,0.1853760083516439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,1,0.5885653495788574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,1,0.5190026760101318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,3,0.5868853330612183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,3,0.49636268615722656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,7,0.6135040124257406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,7,0.49639467398325604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,15,0.5878239870071411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,15,0.496890664100647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,31,0.586847980817159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,31,0.519215981165568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,63,0.5863999923070272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,63,0.5190560022989908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,1,1.1589279969533284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,1,0.9763733545939127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,3,1.1592693328857422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,3,1.0223999818166096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,7,1.1596533457438152
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,7,0.9767253398895264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,15,1.1575840314229329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,15,0.9780426820119222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,31,1.1583999792734783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,31,0.9806773662567139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,3,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,3,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,7,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,7,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,15,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,15,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,31,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,31,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,63,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,63,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,127,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,127,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,255,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,255,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,511,0.03256533294916153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,511,0.02359466751416524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1023,0.052815998593966164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1023,0.03454933315515518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,2047,0.0953653355439504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,2047,0.0583840012550354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,4095,0.17113600174585977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,4095,0.09432533383369446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,8191,0.32233067353566486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,8191,0.16705065965652466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,3,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,3,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,7,0.023168000082174938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,7,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,15,0.023376000424226124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,15,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,31,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,31,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,63,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,63,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,127,0.022416000564893086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,127,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,255,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,255,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,511,0.0322773332397143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,511,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1023,0.04799466828505198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1023,0.043807998299598694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,2047,0.0811466674009959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,2047,0.07230933507283528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,4095,0.13940800229708353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,4095,0.12166933218638103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,63,0.01249066616098086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,511,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,511,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1023,0.014010666559139887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1023,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,2047,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,2047,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,4095,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,4095,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,8191,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,8191,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,16383,0.029930666089057922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,16383,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,31,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,31,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,255,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,511,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1023,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,2047,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,2047,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,4095,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,4095,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,8191,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,8191,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,16383,0.05637866755326589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,16383,0.05012266834576925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1,0.03387200087308884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1,0.031845333675543465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,3,0.03422400106986364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,3,0.033413333197434746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,7,0.03552533437808355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,7,0.03341866781314214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,15,0.033402666449546814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,15,0.0317493329445521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,31,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,31,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,63,0.03463999927043915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,63,0.03405333310365677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,127,0.034629332522551216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,127,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,255,0.03626666714747747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,255,0.031930667658646904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,511,0.05253866811593374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,511,0.04637333254019419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1023,0.08046400050322215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1023,0.07085333267847697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,2047,0.14019200205802917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,2047,0.12192533413569133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,3,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,7,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,7,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,15,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,31,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,31,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,127,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,255,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,255,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,511,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1023,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1023,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,2047,0.023455999791622162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,2047,0.02250666668017705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,4095,0.036015999813874565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,4095,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,8191,0.06038933495680491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,8191,0.05163733164469401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,16383,0.10313066840171814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,16383,0.08901866277058919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1,0.0580213318268458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1,0.05412800113360087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,3,0.05819199979305267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,3,0.05438933273156484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,7,0.05821333328882853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,7,0.05388266841570536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,15,0.057999998331069946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,15,0.05421333511670431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,31,0.0582239975531896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,31,0.05412266651789347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,63,0.0581279993057251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,63,0.053930665055910744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,127,0.05985066791375478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,127,0.05411200225353241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,255,0.05912533402442932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,255,0.05392533540725708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,511,0.08914132912953694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,511,0.08070399860541026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1023,0.1453279952208201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1023,0.12781332929929098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,1,0.10716799894968669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,1,0.09716266393661499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,3,0.10754666725794475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,3,0.09698667128880818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,7,0.10547199845314026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,7,0.09717333316802979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,15,0.10545600454012553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,15,0.09700266520182292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,31,0.105813334385554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,31,0.09690133730570476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,63,0.10567999879519145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,63,0.0965173343817393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,127,0.10579733053843181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,127,0.09890133142471313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,255,0.10534399747848511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,255,0.09692266583442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,1,0.20204265912373862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,1,0.18518932660420737
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,3,0.20197866360346475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,3,0.18587199846903482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,7,0.20163732767105103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,7,0.18517333269119263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,15,0.20175999402999878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,15,0.18529599905014038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,31,0.2015893260637919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,31,0.18525866667429605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,63,0.20161600907643637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,63,0.18521066506703696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,127,0.20125865936279297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,127,0.18555200099945068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,3,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,3,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,7,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,7,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,15,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,31,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,31,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,63,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,63,0.013818666338920593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,127,0.01413333291808764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,127,0.013978666315476099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,255,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,255,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,511,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,511,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1023,0.01960533360640208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1023,0.019578666736682255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,2047,0.03178666780392329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,2047,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,4095,0.050240000089009605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,4095,0.04238399863243103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,8191,0.08295999964078267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,8191,0.07251733541488647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,16383,0.14653866489728293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,16383,0.12779200077056885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,1,0.39000535011291504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,1,0.3596959908803304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,3,0.3901013135910034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,3,0.35763732592264813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,7,0.38863468170166016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,7,0.3590506712595622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,15,0.38997332255045575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,15,0.3592960039774577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,31,0.38998933633168537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,31,0.35955198605855304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,63,0.3886400063832601
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,63,0.35888532797495526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,1,0.7629973093668619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,1,0.7028319835662842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,3,0.764352003733317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,3,0.7026453018188477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,7,0.7647199630737305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,7,0.7025173505147299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,15,0.764293352762858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,15,0.7030986944834391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,31,0.7649546464284261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,31,0.7034453550974528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,3,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,3,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,7,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,7,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,15,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,15,0.015749332805474598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,31,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,31,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,63,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,63,0.01643199970324834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,127,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,127,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,255,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,255,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,511,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,511,0.019461333751678467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1023,0.031498665610949196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1023,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,2047,0.051872000098228455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,2047,0.04562666515509287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,4095,0.08474666873613994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,4095,0.07448000212510426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,8191,0.14814399679501852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,8191,0.12785067160924277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1,0.02348800003528595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,3,0.023530667026837666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,3,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,7,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,7,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,15,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,15,0.023237332701683044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,31,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,31,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,63,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,63,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,127,0.023984000086784363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,127,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,255,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,255,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,511,0.04358933369318644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,511,0.03136533250411352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1023,0.07276266813278198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1023,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,2047,0.13581333557764688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,2047,0.07881600161393483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,4095,0.2521173357963562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,4095,0.1360586682955424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,7,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,15,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,63,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,127,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,255,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,511,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,511,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1023,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1023,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,2047,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,2047,0.015775999675194424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,4095,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,4095,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,8191,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,8191,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,16383,0.04497066636880239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,16383,0.02941333254178365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,31,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,63,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,511,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1023,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1023,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,2047,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,2047,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,4095,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,4095,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,8191,0.05179733534653982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,8191,0.033376000821590424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,16383,0.08950933814048767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,16383,0.05569066603978475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1,0.03719999889532725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,3,0.03774933268626531
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,3,0.033413333197434746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,7,0.03775466730197271
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,7,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,15,0.03782933453718821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,15,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,31,0.037690666814645134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,31,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,63,0.0377866675456365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,63,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,127,0.03950933367013931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,127,0.03335466732581457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,255,0.043791999419530235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,255,0.03561066587766012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,511,0.07361066838105519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,511,0.0499946673711141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1023,0.1281760036945343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1023,0.07688533266385396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,2047,0.24474134047826132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,2047,0.13406933347384134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1,0.011887999872366587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,3,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,3,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,7,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,7,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,15,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,15,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,31,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,31,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,63,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,127,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,127,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,255,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,255,0.012165332833925882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,511,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,511,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1023,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1023,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,2047,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,2047,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,4095,0.056234667698542275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,4095,0.03772266705830892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,8191,0.09532266855239868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,8191,0.05977066854635874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,16383,0.1734079917271932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,16383,0.09999466935793559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1,0.0664160003264745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1,0.05605333546797434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,3,0.06664533416430156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,3,0.056128000219662987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,7,0.06624533236026764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,7,0.05702400207519531
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,15,0.06719466547171275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,15,0.05666666726271311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,31,0.06595199803511302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,31,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,63,0.06486399968465169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,63,0.0565226674079895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,127,0.06668266654014587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,127,0.05635733405749003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,255,0.07268266876538594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,255,0.05653333167235056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,511,0.1313759982585907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,511,0.08692800005276997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1023,0.23914666970570883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1023,0.13819199800491333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,1,0.12113599975903828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,1,0.10312533378601074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,3,0.1209333340326945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,3,0.10314133763313293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,7,0.1218293309211731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,7,0.1032426655292511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,15,0.1218239963054657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,15,0.10347732901573181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,31,0.12191999951998393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,31,0.10315733154614766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,63,0.1216319998105367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,63,0.10358400146166484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,127,0.11994666854540507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,127,0.10354133447011311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,255,0.1318719983100891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,255,0.10326400399208069
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,1,0.22849067052205405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,1,0.19550400972366333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,3,0.2288693388303121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,3,0.19564266999562582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,7,0.23042666912078857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,7,0.1954666574796041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,15,0.2291146715482076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,15,0.1954773267110189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,31,0.23001599311828613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,31,0.19580266873041788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,63,0.22956266005833945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,63,0.1961066722869873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,127,0.22650132576624551
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,127,0.1941546599070231
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,3,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,3,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,7,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,7,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,15,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,15,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,31,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,31,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,63,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,63,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,127,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,127,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,255,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,255,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,511,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,511,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1023,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1023,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,2047,0.049253334601720176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,2047,0.03271466741959254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,4095,0.07865599791208903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,4095,0.04990399877230326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,8191,0.14240533113479614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,8191,0.08099733293056488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,16383,0.26714134216308594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,16383,0.1421280006567637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,1,0.44333334763844806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,1,0.375274658203125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,3,0.4431360165278117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,3,0.3759146531422933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,7,0.44328534603118896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,7,0.37547731399536133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,15,0.4428746700286865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,15,0.3758186499277751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,31,0.4432640075683594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,31,0.37577064832051593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,63,0.4433813492457072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,63,0.37562131881713867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,1,0.873306671778361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,1,0.7378719647725424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,3,0.872208038965861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,3,0.7372960249582926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,7,0.8715999921162924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,7,0.7362186908721924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,15,0.8716746966044108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,15,0.7369013627370199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,31,0.8717119693756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,31,0.7664106686909994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,3,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,3,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,7,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,7,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,15,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,15,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,31,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,31,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,63,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,127,0.01773333301146825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,127,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,255,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,255,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,511,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,511,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1023,0.04585599899291992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1023,0.029338667790095013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,2047,0.07976533472537994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,2047,0.049973333875338234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,4095,0.14261866609255472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,4095,0.08078399797280629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,8191,0.26926400264104206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,8191,0.144405335187912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1,0.02083733429511388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,3,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,3,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,7,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,7,0.020400000115235645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,15,0.02046400060256322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,15,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,31,0.020762667059898376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,31,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,63,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,63,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,127,0.02081599955757459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,127,0.019973333925008774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,255,0.020831999679406483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,255,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,511,0.028981332977612812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,511,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1023,0.04162666698296865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1023,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,2047,0.06909333169460297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,2047,0.06203199923038483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,4095,0.11781332890192668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,4095,0.1030613382657369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,7,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,63,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,127,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,127,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,255,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1023,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,2047,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,2047,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,4095,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,4095,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,8191,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,8191,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,16383,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,16383,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,7,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1023,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,2047,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,2047,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,4095,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,4095,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,8191,0.033530667424201965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,8191,0.02934933453798294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,16383,0.05717866619427999
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,16383,0.04933866858482361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1,0.03030933439731598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1,0.029690665503342945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,3,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,3,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,7,0.029711998999118805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,7,0.0283146674434344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,15,0.029829333225886028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,15,0.02992533395687739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,31,0.02974933385848999
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,31,0.029370665550231934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,63,0.029535998900731403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,63,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,127,0.029530666768550873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,127,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,255,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,255,0.02734400083621343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,511,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,511,0.03961066653331121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1023,0.06836799780527751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1023,0.061994666854540505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,2047,0.11785067121187846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,2047,0.10326400399208069
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,3,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,3,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,31,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,63,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,127,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,127,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,255,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,255,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,511,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,511,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1023,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1023,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,2047,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,2047,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,4095,0.03573866685231527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,4095,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,8191,0.06066666543483734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,8191,0.05009600023428599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,16383,0.10442133744557698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,16383,0.09011200070381165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1,0.049653331438700356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1,0.046469335754712425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,3,0.05027733246485392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,3,0.04577599962552389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,7,0.04991999765237173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,7,0.0459146648645401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,15,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,15,0.04586666822433472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,31,0.04997866849104563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,31,0.04708800216515859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,63,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,63,0.04744533201058706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,127,0.050144001841545105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,127,0.04593066871166229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,255,0.05193066596984863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,255,0.04721599817276001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,511,0.07653333246707916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,511,0.07002666592597961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1023,0.12156266967455547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1023,0.10742933551470439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,1,0.08957866827646892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,1,0.08271466692288716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,3,0.08986666798591614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,3,0.08258666594823201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,7,0.09098133444786072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,7,0.08261333405971527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,15,0.08954133590062459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,15,0.08267199993133545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,31,0.09025067090988159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,31,0.08251733581225078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,63,0.09057066837946574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,63,0.08243733147780101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,127,0.09086400270462036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,127,0.08263466755549113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,255,0.08888000249862671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,255,0.08263466755549113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,1,0.17061867316563925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,1,0.15642133355140686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,3,0.17082132895787558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,3,0.15657066305478415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,7,0.16874132553736368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,7,0.1553920010725657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,15,0.1693333387374878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,15,0.15520532925923666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,31,0.1693440079689026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,31,0.15628266334533691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,63,0.16876800855000815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,63,0.15446399648984274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,127,0.16916799545288086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,127,0.15734400351842245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,3,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,3,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,7,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,7,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,15,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,15,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,31,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,31,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,63,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,63,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,127,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,127,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,255,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,255,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,511,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,511,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1023,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1023,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,2047,0.030037333567937214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,2047,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,4095,0.050554667909940086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,4095,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,8191,0.08281599978605907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,8191,0.07237333556016286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,16383,0.14619200428326926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,16383,0.12574932972590128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,1,0.32638933261235553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,1,0.301749328772227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,3,0.32638933261235553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,3,0.30086400111516315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,7,0.3263946572939555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,7,0.29973334074020386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,15,0.3266826669375102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,15,0.30038400491078693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,31,0.3264853358268738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,31,0.3009653290112813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,63,0.3261173367500305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,63,0.29982932408650714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,1,0.6380373239517212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,1,0.5877013206481934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,3,0.6381706794102987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,3,0.5872480074564616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,7,0.6383253335952759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,7,0.5867146650950114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,15,0.6379839976628622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,15,0.5876266558965048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,31,0.6383999983469645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,31,0.5879573424657186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,3,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,3,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,7,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,7,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,15,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,15,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,31,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,31,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,63,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,63,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,127,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,127,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,255,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,255,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,511,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,511,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1023,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1023,0.023621333142121632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,2047,0.046069333950678505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,2047,0.041536000867684685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,4095,0.07445333401362102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,4095,0.0641546646753947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,8191,0.1271573305130005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,8191,0.10947733124097188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1,0.02311466634273529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,3,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,3,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,7,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,7,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,15,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,15,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,31,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,31,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,63,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,63,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,127,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,127,0.019461333751678467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,255,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,255,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,511,0.03858133405447006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,511,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1023,0.06243200103441874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1023,0.04178133110205332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,2047,0.115365336338679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,2047,0.06805866460005443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,4095,0.2097760041554769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,4095,0.11610666910807292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,3,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,3,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,7,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,31,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,511,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1023,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1023,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,2047,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,4095,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,4095,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,8191,0.02350933353106181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,8191,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,16383,0.04162666698296865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,16383,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,63,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,511,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1023,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1023,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,2047,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,2047,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,4095,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,4095,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,8191,0.05062933266162872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,8191,0.033386667569478355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,16383,0.08673066894213359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,16383,0.05463466544946035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,3,0.03317866722742716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,3,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,7,0.033600000043710075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,7,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,15,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,15,0.02935466667016347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,31,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,31,0.029274667302767437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,63,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,63,0.029477333029111225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,127,0.03499733408292135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,127,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,255,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,255,0.03141333411137263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,511,0.06334400177001953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,511,0.043765331308046974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1023,0.10729066530863444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1023,0.06559466818968455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,2047,0.20438933372497559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,2047,0.11352533102035522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,3,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,7,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,7,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,15,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,15,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,31,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,31,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,63,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,127,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,127,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,255,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,511,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,511,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1023,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1023,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,2047,0.030245333909988403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,2047,0.024186665813128155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,4095,0.05593066910902659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,4095,0.03578133384386698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,8191,0.0942080020904541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,8191,0.05829866727193197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,16383,0.17157334089279175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,16383,0.09928533434867859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1,0.05657599866390228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1,0.04818133513132731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,3,0.0561706672112147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,3,0.04807466765244802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,7,0.058176000912984215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,7,0.047872001926104225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,15,0.05794133245944977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,15,0.04808000226815542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,31,0.05640000104904175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,31,0.04774933556715647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,63,0.056176001826922096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,63,0.04806933303674062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,127,0.05759466687838236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,127,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,255,0.06227200229962667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,255,0.049733335773150124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,511,0.1109920044740041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,511,0.07435200115044911
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1023,0.2015786568323771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1023,0.11743467052777608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,1,0.10331733028093974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,1,0.08710933725039165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,3,0.10307733217875163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,3,0.08692800005276997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,7,0.10308266679445903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,7,0.08861333131790161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,15,0.1032319962978363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,15,0.08875200152397156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,31,0.10331199566523235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,31,0.08875200152397156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,63,0.10315199693044026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,63,0.08854400118192036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,127,0.10326932867368062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,127,0.08874666690826416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,255,0.11132799585660298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,255,0.08690133690834045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,1,0.19285867611567178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,1,0.1641386648019155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,3,0.19341333707173666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,3,0.16454399625460306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,7,0.19354132811228433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,7,0.16474133729934692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,15,0.19348265727361044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,15,0.16471466422080994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,31,0.1930933396021525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,31,0.16459733247756958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,63,0.19325333833694458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,63,0.1648906668027242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,127,0.19125866889953613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,127,0.1639359990755717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,3,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,3,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,7,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,7,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,15,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,15,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,31,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,31,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,63,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,63,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,127,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,127,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,255,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,511,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,511,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1023,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1023,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,2047,0.04762666424115499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,2047,0.03121600051720937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,4095,0.07737599809964497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,4095,0.04966933528582255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,8191,0.1381760040918986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,8191,0.0820853312810262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,16383,0.26073066393534344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,16383,0.14217600226402283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,1,0.3715200026830037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,1,0.31624533732732135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,3,0.37170132001241046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,3,0.3160533308982849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,7,0.371509313583374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,7,0.31625600655873615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,15,0.370746652285258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,15,0.3158133427302043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,31,0.3710346619288127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,31,0.3152373234430949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,63,0.37165868282318115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,63,0.3165866732597351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,1,0.7280693054199219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,1,0.6166026592254639
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,3,0.7286826769510905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,3,0.6162346601486206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,7,0.7286400000254313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,7,0.61735467116038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,15,0.7272960344950358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,15,0.6169493198394775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,31,0.7273066838582357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,31,0.6157653331756592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,3,0.01551466683546702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,3,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,7,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,7,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,15,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,15,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,31,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,31,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,63,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,63,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,127,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,127,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,255,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,255,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,511,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,511,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1023,0.039781334499518074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1023,0.0269813338915507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,2047,0.0692799985408783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,2047,0.04558399816354116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,4095,0.12219732999801636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,4095,0.07212266822655995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,8191,0.2276960015296936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,8191,0.12358400225639343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,3,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,3,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,7,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,7,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,15,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,15,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,31,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,31,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,63,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,63,0.018522666146357853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,127,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,127,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,255,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,255,0.01764800027012825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,511,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,511,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1023,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1023,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,2047,0.058186665177345276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,2047,0.05193066596984863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,4095,0.09715200463930766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,4095,0.08478933572769165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,8191,0.17068266868591309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,8191,0.14801599582036337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,63,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,63,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,255,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,511,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,2047,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,2047,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,4095,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,4095,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,8191,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,8191,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,16383,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,16383,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,32767,0.03549866626660029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,32767,0.02938133229811986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,3,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,127,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,127,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,255,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,511,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,511,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1023,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1023,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,2047,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,4095,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,8191,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,8191,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,16383,0.037461332976818085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,16383,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,32767,0.05881066620349884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,32767,0.05161066850026449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,3,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,3,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,7,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,7,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,15,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,15,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,31,0.025493333737055462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,31,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,63,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,63,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,127,0.027109332382678986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,127,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,255,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,255,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,511,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,511,0.03377600014209747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1023,0.05807466804981232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1023,0.05218133330345154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,2047,0.09904533624649048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,2047,0.08682133754094441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,4095,0.1738133430480957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,4095,0.1502400040626526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,511,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,511,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1023,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,2047,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,2047,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,4095,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,4095,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,8191,0.03944533318281174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,8191,0.03142400085926056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,16383,0.059936001896858215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,16383,0.05410666763782501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,32767,0.10307199756304423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,32767,0.08881066242853801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1,0.04171200096607208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1,0.03951466580231985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,3,0.04193066557248434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,3,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,7,0.04158399999141693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,7,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,15,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,15,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,31,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,31,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,63,0.04142399877309799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,63,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,127,0.041834667325019836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,127,0.039146666725476585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,255,0.043391997615496315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,255,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,511,0.06421866516272227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,511,0.05791999896367391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1023,0.10122133294741313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1023,0.08874666690826416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,2047,0.17682133118311563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,2047,0.15410666664441428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,1,0.07295466462771098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,1,0.06678933401902516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,3,0.07267199953397115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,3,0.06656533479690552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,7,0.07284800211588542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,7,0.06735466420650482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,15,0.07334933181603749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,15,0.06714133421579997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,31,0.07453333338101704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,31,0.06862399975458781
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,63,0.0746666689713796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,63,0.06865066786607106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,127,0.07461866736412048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,127,0.06815466781457265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,255,0.07467199862003326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,255,0.06853866577148438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,511,0.11422933141390483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,511,0.10234133402506511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,1,0.13777066270510355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,1,0.12562132875124613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,3,0.13870933651924133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,3,0.1256053348382314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,7,0.1379039982954661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,7,0.12573333581288657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,15,0.13831999897956848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,15,0.12543466687202454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,31,0.13807466626167297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,31,0.1257973313331604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,63,0.13807466626167297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,63,0.12583999832471213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,127,0.136543999115626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,127,0.1258080005645752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,255,0.1356160044670105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,255,0.1236853301525116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,3,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,7,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,7,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,15,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,15,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,31,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,63,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,63,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,127,0.012154666086037954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,127,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,255,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,255,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,511,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,511,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1023,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1023,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,2047,0.024138666689395905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,2047,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,4095,0.04008533308903376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,4095,0.03246400008598963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,8191,0.06091199815273285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,8191,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,16383,0.10306666294733684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,16383,0.09070400396982829
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,32767,0.18973867098490396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,32767,0.16265599926312765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,1,0.26290132602055866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,1,0.24260266621907553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,3,0.26310400168100995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,3,0.24263467391331991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,7,0.26294400294621784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,7,0.24265599250793457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,15,0.2630400061607361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,15,0.24277333418528238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,31,0.26285332441329956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,31,0.24313066403071085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,63,0.2634613315264384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,63,0.24263467391331991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,127,0.26258667310078937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,127,0.2423093318939209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,1,0.514298677444458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,1,0.47359466552734375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,3,0.5133813222249349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,3,0.4736800193786621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,7,0.5136266549428304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,7,0.47303466002146405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,15,0.5144000053405762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,15,0.4739146629969279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,31,0.5145973364512125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,31,0.47388799985249835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,63,0.5138026475906372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,63,0.4726506471633911
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,3,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,3,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,7,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,7,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,15,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,15,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,31,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,31,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,63,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,63,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,127,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,127,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,255,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,255,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,511,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,511,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1023,0.02402133246262868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1023,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,2047,0.04148799926042557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,2047,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,4095,0.06233066817124685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,4095,0.05422399938106537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,8191,0.10528533657391866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,8191,0.09090666969617207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,16383,0.19051200151443481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,16383,0.16325333714485168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,3,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,3,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,7,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,7,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,15,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,15,0.017680000513792038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,31,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,31,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,63,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,63,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,127,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,127,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,255,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,255,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,511,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,511,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1023,0.05213333169619242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1023,0.035530666510264076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,2047,0.09404266873995464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,2047,0.0580266664425532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,4095,0.09479467074076335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,4095,0.1701386570930481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,8191,0.16723199685414633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,8191,0.3222986658414205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,15,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,63,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,255,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,511,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,511,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,2047,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,4095,0.016672000288963318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,4095,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,8191,0.020821332931518555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,8191,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,16383,0.0352906659245491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,16383,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,32767,0.056128000219662987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,32767,0.03756266583998998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,3,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,63,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,511,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1023,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1023,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,2047,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,2047,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,4095,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,4095,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,8191,0.035605333745479584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,8191,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,16383,0.05739733576774597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,16383,0.037733333806196846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,32767,0.09807466467221577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,32767,0.058143998185793556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,3,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,3,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,7,0.02939733366171519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,7,0.02584533393383026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,15,0.029178666571776073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,15,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,31,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,31,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,63,0.029109333952267964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,63,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,127,0.029205332199732464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,127,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,255,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,255,0.025381334125995636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,511,0.05423999826113383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,511,0.03791466603676478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1023,0.09115200241406758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1023,0.056159997979799904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,2047,0.16914133230845133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,2047,0.09577600161234538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,4095,0.16915732622146606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,4095,0.3205440044403076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,3,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,63,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,511,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1023,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1023,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,2047,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,2047,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,4095,0.0377813329299291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,4095,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,8191,0.059119999408721924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,8191,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,16383,0.09967466195424397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,16383,0.05834133426348368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,32767,0.1820746660232544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,32767,0.09894399841626485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1,0.046944002310434975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,7,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,3,0.04866666595141093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,3,0.04197866717974345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,7,0.04633066554864248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,7,0.04170133173465729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,15,0.04784533381462097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,15,0.041573333243529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,31,0.048058668772379555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,31,0.04137066751718521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,63,0.04776533444722494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,63,0.041573333243529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,127,0.04972266654173533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,127,0.041536000867684685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,255,0.05357866485913595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,255,0.0421973317861557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,511,0.09271466732025146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,511,0.06205866734186808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1023,0.16482667128245035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1023,0.09659199913342793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,2047,0.3185013333956401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,2047,0.17044266064961752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,1,0.0845973292986552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,1,0.07064533233642578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,3,0.08473599950472514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,3,0.07029333213965099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,7,0.08470933636029561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,15,0.08481066425641377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,15,0.07257600128650665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,31,0.0848533312479655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,31,0.07215466598669688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,63,0.08474666873613994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,63,0.07138133545716603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,127,0.08424533406893413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,127,0.07268266876538594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,255,0.09130133191744487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,255,0.07262399792671204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,511,0.16673600673675537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,511,0.11104533076286316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,1,0.1566986640294393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,1,0.13408000270525613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,3,0.1570186714331309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,3,0.1341600020726522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,7,0.15706132849057516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,7,0.13448533415794373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,15,0.15705600380897522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,15,0.1349386672178904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,31,0.15710933009783426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,31,0.13414399822553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,63,0.1565013329188029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,63,0.13404267032941183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,127,0.1557973325252533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,127,0.13448533415794373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,255,0.1700800061225891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,255,0.1323946714401245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,3,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,3,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,7,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,7,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,15,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,31,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,31,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,63,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,63,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,127,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,255,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,255,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,511,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,511,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1023,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1023,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,2047,0.0378560001651446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,2047,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,4095,0.05957333246866862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,4095,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,8191,0.10074133674303691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,8191,0.060085331400235496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,16383,0.18374399344126383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,16383,0.10147200028101604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,32767,0.3498186667760213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,32767,0.18363734086354574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,1,0.29983999331792194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,1,0.2556426723798116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,3,0.2998879949251811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,3,0.2569653391838074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,7,0.3008906642595927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,7,0.2568853298823039
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,15,0.30053865909576416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,15,0.25593600670496625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,31,0.3001120090484619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,31,0.25573333104451496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,63,0.299834668636322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,63,0.2567360003789266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,127,0.29870933294296265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,127,0.2563893397649129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,1,0.5859839916229248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,1,0.4974079926808675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,3,0.5856853326161703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,3,0.4972533384958903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,7,0.5853333473205566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,7,0.49641064802805585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,15,0.5847839911778768
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,15,0.4969866673151652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,31,0.6150240103403727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,31,0.49668268362681073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,63,0.6357653141021729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,63,0.49722667535146076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,3,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,3,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,7,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,7,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,15,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,15,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,31,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,31,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,63,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,63,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,127,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,127,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,255,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,255,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,511,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,511,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1023,0.03396799912055334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1023,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,2047,0.060271998246510826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,2047,0.039749334255854286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,4095,0.10116266210873921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,4095,0.061664000153541565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,8191,0.18418665726979574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,8191,0.10338133573532104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,16383,0.35151998202006024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,16383,0.1860319972038269
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,3,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,3,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,7,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,7,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,15,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,15,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,31,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,31,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,63,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,63,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,127,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,127,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,255,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,255,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,511,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,511,0.019637333850065868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1023,0.031680000325044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1023,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,2047,0.05253866811593374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,2047,0.04552533229192098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,4095,0.08457600076993306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,4095,0.07503466804822286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,8191,0.14857066671053568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,8191,0.12820800145467123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,3,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,3,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,7,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,7,0.01249066616098086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,63,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,127,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,255,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,255,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1023,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,2047,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,2047,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,4095,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,4095,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,8191,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,8191,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,16383,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,16383,0.019551999866962433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,32767,0.030661332110563915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,32767,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,31,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,511,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1023,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1023,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,2047,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,2047,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,4095,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,4095,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,8191,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,8191,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,16383,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,16383,0.025439999997615814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,32767,0.0465280016263326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,32767,0.04055466751257578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1,0.021744000415007275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,3,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,3,0.022181332111358643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,7,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,7,0.02163200080394745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,15,0.023344000180562336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,15,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,31,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,31,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,63,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,63,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,127,0.022944000860055287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,127,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,255,0.02197333425283432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,255,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,511,0.03369066615899404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,511,0.02939733366171519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1023,0.047872001926104225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1023,0.042117332418759666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,2047,0.08265600105126698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,2047,0.07320533196131389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,4095,0.14060266812642416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,4095,0.1216213305791219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,7,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,63,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,511,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1023,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1023,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,2047,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,2047,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,4095,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,4095,0.021557333568731945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,8191,0.03540800015131632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,8191,0.03075733284155528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,16383,0.059445331494013466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,16383,0.05011733373006185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,32767,0.10187199711799622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,32767,0.08906132976214091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1,0.03341866781314214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,3,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,3,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,7,0.03541333228349686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,7,0.0312266672650973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,15,0.03388266762097677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,15,0.0317493329445521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,31,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,31,0.03325333446264267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,63,0.035242666800816856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,63,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,127,0.03530666728814443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,127,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,255,0.03565333286921183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,255,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,511,0.052149335543314614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,511,0.04607999821503957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1023,0.08041599889596303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1023,0.07051733136177063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,2047,0.13988799850145975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,2047,0.1218293309211731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,1,0.0581226646900177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,1,0.05379733443260193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,3,0.057850668827692665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,3,0.05403199791908264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,7,0.058149332801500954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,7,0.05414933462937673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,15,0.058037335673967995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,15,0.05418133238951365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,31,0.05816533168156942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,31,0.054144000013669334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,63,0.05835199852784475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,63,0.05403733253479004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,127,0.058389330903689064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,127,0.054005334774653115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,255,0.05890666445096334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,255,0.05409066875775655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,511,0.09165866176287334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,511,0.08099733293056488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,1,0.10725333293279012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,1,0.09681600332260132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,3,0.10540266831715901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,3,0.0972053309281667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,7,0.10548800230026245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,7,0.09709333380063374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,15,0.10621333122253418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,15,0.09697600205739339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,31,0.10577600200970967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,31,0.0974026620388031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,63,0.10589333375295003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,63,0.09691199660301208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,127,0.10706133643786113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,127,0.09710933764775594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,255,0.10656000177065532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,255,0.09726400176684062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,3,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,3,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,7,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,7,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,15,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,15,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,31,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,31,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,63,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,127,0.012362666428089142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,127,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,255,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,255,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,511,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,511,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1023,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1023,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,2047,0.023034666975339253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,2047,0.02294933299223582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,4095,0.03608000030120214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,4095,0.03169599920511246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,8191,0.060559997955958046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,8191,0.05192000170548757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,16383,0.10410133004188538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,16383,0.09089600046475728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,32767,0.18950400749842325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,32767,0.16267733772595724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,1,0.20247467358907065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,1,0.18701332807540894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,3,0.20279467105865479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,3,0.1850879987080892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,7,0.20168532927831015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,7,0.18564265966415405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,15,0.20193066199620566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,15,0.18528000513712564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,31,0.20176533857981363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,31,0.18547199169794717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,63,0.20163732767105103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,63,0.18676799535751343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,127,0.201690673828125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,127,0.1869866649309794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,1,0.39006932576497394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,1,0.35786668459574383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,3,0.3901066780090332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,3,0.35836267471313477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,7,0.39005335172017414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,7,0.3593706687291463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,15,0.3900586764017741
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,15,0.3590986728668213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,31,0.3901173273722331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,31,0.35894934336344403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,63,0.39001067479451496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,63,0.35894401868184406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,3,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,3,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,7,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,7,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,15,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,15,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,31,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,31,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,63,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,63,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,127,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,127,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,255,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,255,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,511,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1023,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1023,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,2047,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,2047,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,4095,0.051781331499417625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,4095,0.04403733213742574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,8191,0.08332799871762593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,8191,0.07218133409818013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,16383,0.1471946636835734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,16383,0.1264639993508657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,3,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,3,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,7,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,7,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,15,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,15,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,31,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,31,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,63,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,63,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,127,0.018565333137909572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,127,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,255,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,255,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,511,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,511,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1023,0.04463466505209605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1023,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,2047,0.08087466657161713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,2047,0.05178666611512502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,4095,0.14380266269048056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,4095,0.0832533339659373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,8191,0.2699413299560547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,8191,0.14441066980361938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,3,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,3,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,7,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,7,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,15,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,31,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,31,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,63,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,255,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,511,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1023,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,2047,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,2047,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,4095,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,4095,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,8191,0.01966933285196622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,8191,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,16383,0.025631998976071674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,16383,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,32767,0.04647466540336609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,32767,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,15,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,127,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,255,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1023,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1023,0.013536000003417334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,2047,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,2047,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,4095,0.019658666104078293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,4095,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,8191,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,8191,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,16383,0.047370667258898415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,16383,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,32767,0.07529599964618683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,32767,0.045925334095954895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1,0.024405332903067272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,3,0.024890666206677754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,3,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,7,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,7,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,15,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,15,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,31,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,31,0.021759999295075733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,63,0.024842667082945507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,63,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,127,0.023898666103680927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,127,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,255,0.026789332429567974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,255,0.023024000227451324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,511,0.04525866607824961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,511,0.03169066707293192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1023,0.07351999978224437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1023,0.04756799836953481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,2047,0.13597866892814636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,2047,0.07977599898974101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,4095,0.25329599777857464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,4095,0.13588800032933554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,3,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,127,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,511,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1023,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1023,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,2047,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,2047,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,4095,0.032314665615558624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,4095,0.023765332996845245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,8191,0.054757331808408104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,8191,0.037589333951473236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,16383,0.0925386647383372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,16383,0.05807999769846598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,32767,0.16750399271647134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,32767,0.0972053309281667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,3,0.039162665605545044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,3,0.03386666625738144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,7,0.03842666745185852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,7,0.03379733363787333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,15,0.03889599939187368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,15,0.03365866591533025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,31,0.03868799904982249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,31,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,63,0.037845333417256675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,63,0.03356266766786575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,127,0.039893334110577904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,127,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,255,0.0439626673857371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,255,0.035418666899204254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,511,0.07447466750939687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,511,0.0498933345079422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1023,0.1279039978981018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1023,0.0767626663049062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,2047,0.24512000878651938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,2047,0.13588266571362814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,1,0.06656000018119812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,1,0.0561706672112147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,3,0.06648000081380208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,3,0.056330665946006775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,7,0.06656000018119812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,7,0.056261335810025535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,15,0.06704533100128174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,15,0.056133334835370384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,31,0.06653866668542226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,31,0.056074668963750206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,63,0.06613333523273468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,63,0.056261335810025535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,127,0.06658666829268138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,127,0.058330665032068886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,255,0.07246933380762736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,255,0.05820266902446747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,511,0.130730668703715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,511,0.08695466319719951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,1,0.12160533666610718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,1,0.1033066709836324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,3,0.12118400136629741
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,3,0.10458133618036906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,7,0.12166399757067363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,7,0.10515200098355611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,15,0.12187199791272481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,15,0.10538132985432942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,31,0.12185600399971008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,31,0.10485866665840149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,63,0.12180266777674358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,63,0.10539733370145161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,127,0.11983999609947205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,127,0.10517332951227824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,255,0.13195199767748514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,255,0.10539733370145161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,3,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,7,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,7,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,15,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,31,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,31,0.011994666109482447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,63,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,63,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,127,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,127,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,255,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,255,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,511,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,511,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1023,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1023,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,2047,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,2047,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,4095,0.05597866574923197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,4095,0.03736000011364619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,8191,0.09557867050170898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,8191,0.059903999169667564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,16383,0.17441600561141968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,16383,0.10125333070755005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,32767,0.331498662630717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,32767,0.1832053263982137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,1,0.22826133171717325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,1,0.1971199909845988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,3,0.22936000426610312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,3,0.19627734025319418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,7,0.22793600956598917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,7,0.19565866390864053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,15,0.22831465800603232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,15,0.19556266069412231
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,31,0.22800000508626303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,31,0.19564266999562582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,63,0.228112002213796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,63,0.19683732589085898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,127,0.2272746761639913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,127,0.19778132438659668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,1,0.44356799125671387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,1,0.379861315091451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,3,0.4440053304036458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,3,0.3797173500061035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,7,0.4434933265050252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,7,0.40805331865946454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,15,0.4433279832204183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,15,0.37857600053151447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,31,0.4429173469543457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,31,0.379472017288208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,63,0.4437013467152913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,63,0.40850667158762616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,3,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,3,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,7,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,7,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,15,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,15,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,31,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,31,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,63,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,63,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,127,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,127,0.013823999712864557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,255,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,511,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,511,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1023,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1023,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,2047,0.04966400067011515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,2047,0.03183466692765554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,4095,0.08049599826335907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,4095,0.0499946673711141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,8191,0.14291200041770935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,8191,0.08191466828187306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,16383,0.26922667026519775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,16383,0.144405335187912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,3,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,3,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,7,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,15,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,15,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,31,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,31,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,63,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,63,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,127,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,127,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,255,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,255,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,511,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,511,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1023,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1023,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,2047,0.04067199925581614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,2047,0.03344533344109853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,4095,0.06303999821345012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,4095,0.05596800148487091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,8191,0.1053013304869334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,8191,0.09171199798583984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,16383,0.19153066476186117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,16383,0.16476800044377646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,3,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,31,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1023,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1023,0.012330666184425354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,2047,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,2047,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,4095,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,4095,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,8191,0.017674667139848072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,8191,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,16383,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,16383,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,32767,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,32767,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,65535,0.03745600084463755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,65535,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,3,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,511,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,2047,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,2047,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,4095,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,4095,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,8191,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,8191,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,16383,0.023749334116776783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,16383,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,32767,0.037621334195137024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,32767,0.031125334401925404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,65535,0.06065600117047628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,65535,0.051728000243504844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,3,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,3,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,7,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,7,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,15,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,15,0.018698666244745255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,31,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,31,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,63,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,63,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,127,0.018437333405017853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,127,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,255,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,255,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,511,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,511,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1023,0.03562666724125544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1023,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,2047,0.05946133534113566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,2047,0.0531626691420873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,4095,0.09699199597040813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,4095,0.08471999565760295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,8191,0.1716746687889099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,8191,0.1464906632900238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,7,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,255,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,511,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,511,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1023,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,2047,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,2047,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,4095,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,4095,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,8191,0.025018667181332905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,8191,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,16383,0.03988266736268997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,16383,0.033215999603271484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,32767,0.06253866851329803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,32767,0.05453333258628845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,65535,0.10699199636777242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,65535,0.08860266208648682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,3,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,3,0.02493866781393687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,7,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,15,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,15,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,31,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,31,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,63,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,63,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,127,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,127,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,255,0.02605333427588145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,255,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,511,0.039781334499518074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,511,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1023,0.05841066439946493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1023,0.05216533442338308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,2047,0.10060266653696696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,2047,0.08847999572753906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,4095,0.17476266622543335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,4095,0.1504800021648407
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1,0.04190400242805481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,3,0.042021334171295166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,3,0.039434666434923805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,7,0.04159999887148539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,7,0.039594667653242745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,15,0.041936000188191734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,15,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,31,0.0417546679576238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,31,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,63,0.041749333341916404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,63,0.03994133323431015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,127,0.04368533194065094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,127,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,255,0.0439573327700297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,255,0.03916800022125244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,511,0.06410666803518932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,511,0.058304001887639366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1023,0.10031466682751973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1023,0.08909866213798523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,1,0.07437866429487865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,1,0.06855999926726024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,3,0.07452799876530965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,3,0.06800533334414165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,7,0.07459733386834462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,7,0.06817066669464111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,15,0.07436800003051758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,15,0.0683840016523997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,31,0.07435733576615651
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,31,0.0682666649421056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,63,0.07471466561158498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,63,0.06846400101979573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,127,0.0751039981842041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,127,0.06829333305358887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,255,0.07509333391984303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,255,0.06863466898600261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,511,0.1153600017229716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,511,0.10283199946085612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,7,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,31,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,511,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,2047,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,2047,0.01782400036851565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,4095,0.024735999604066212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,4095,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,8191,0.039359999199708305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,8191,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,16383,0.060309335589408875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,16383,0.05298133194446564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,32767,0.10326932867368062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,32767,0.0888853371143341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,65535,0.18903466065724692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,65535,0.1612266699473063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,1,0.13820266723632812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,1,0.12610666950543722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,3,0.1381760040918986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,3,0.1265120009581248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,7,0.13823999961217245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,7,0.12589866916338602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,15,0.138373335202535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,15,0.12617599964141846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,31,0.1381439963976542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,31,0.12591999769210815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,63,0.1381706694761912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,63,0.1256106694539388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,127,0.13703466455141702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,127,0.1278986632823944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,255,0.1362559994061788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,255,0.12479466199874878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,1,0.26507200797398883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,1,0.24265066782633463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,3,0.26364799340566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,3,0.2425439953804016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,7,0.2641493280728658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,7,0.24276266495386759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,15,0.26358399788538617
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,15,0.2426933248837789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,31,0.26383999983469647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,31,0.2430880069732666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,63,0.2648959954579671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,63,0.24407466252644858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,127,0.26316267251968384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,127,0.2428106665611267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,3,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,3,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,7,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,7,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,15,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,15,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,31,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,31,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,63,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,63,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,127,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,255,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,511,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1023,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1023,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,2047,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,2047,0.023386667172114056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,4095,0.039962666730086006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,4095,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,8191,0.06248533229033152
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,8191,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,16383,0.10548800230026245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,16383,0.09053333600362141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,32767,0.18950400749842325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,32767,0.16243732968966165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,3,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,3,0.016496000190575916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,7,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,7,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,15,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,15,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,31,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,31,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,63,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,63,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,127,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,127,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,255,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,255,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,511,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,511,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1023,0.03340800106525421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1023,0.024869332710901897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,2047,0.060191998879114784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,2047,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,4095,0.1014453371365865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,4095,0.06084266801675161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,8191,0.18398400147755942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,8191,0.10248000423113506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,16383,0.34989333152770996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,16383,0.18587199846903482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,31,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,31,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,255,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,511,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,511,0.012373333175977072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1023,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1023,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,2047,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,2047,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,4095,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,4095,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,8191,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,8191,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,16383,0.02260799954334895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,16383,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,32767,0.03758399933576584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,32767,0.024959998826185863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,65535,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,65535,0.037685332198937736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,15,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,31,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,127,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1023,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1023,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,2047,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,2047,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,4095,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,4095,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,8191,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,8191,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,16383,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,16383,0.023168000082174938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,32767,0.05633600056171417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,32767,0.037621334195137024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,65535,0.0974720021088918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,65535,0.05761066575845083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,3,0.020549333343903225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,3,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,7,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,7,0.017525333911180496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,15,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,15,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,31,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,31,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,63,0.02090666691462199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,63,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,127,0.019930666933457058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,127,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,255,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,255,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,511,0.03377600014209747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,511,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1023,0.05226133267084757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1023,0.03474666674931844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,2047,0.09459199508031209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,2047,0.058389330903689064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,4095,0.17057067155838013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,4095,0.09523733456929524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,8191,0.32108267148335773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,8191,0.16809600591659546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,511,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1023,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1023,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,2047,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,2047,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,4095,0.021530665457248688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,4095,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,8191,0.03861333429813385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,8191,0.02585600068171819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,16383,0.06032533446947733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,16383,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,32767,0.10021332899729411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,32767,0.06041066845258077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,65535,0.18268267313639322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,65535,0.10175999999046326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1,0.029418667157491047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1,0.0259253333012263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,3,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,3,0.025562666356563568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,7,0.029002666473388672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,7,0.02611200014750163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,15,0.029472000896930695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,15,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,31,0.0296426663796107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,31,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,63,0.02935466667016347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,63,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,127,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,127,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,255,0.033285332222779594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,255,0.026288000245889027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,511,0.05409066875775655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,511,0.03941866755485535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1023,0.09106666843096416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1023,0.05665066838264465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,2047,0.1704746683438619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,2047,0.09774399797121684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,4095,0.32172266642252606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,4095,0.17061332861582437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1,0.04770666857560476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1,0.03965866565704346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,3,0.04785066843032837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,3,0.041402667760849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,7,0.048122664292653404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,7,0.04151466737190882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,15,0.047781333327293396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,15,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,31,0.047781333327293396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,31,0.040021332601706185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,63,0.047557334105173744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,63,0.03997866561015447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,127,0.047914668917655945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,127,0.04115733255942663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,255,0.05243733525276184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,255,0.04261333247025808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,511,0.09283733367919922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,511,0.06251200040181477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1023,0.16529599825541177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1023,0.09804800152778625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,1,0.08503466844558716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,1,0.07250133156776428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,3,0.08492799599965413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,3,0.07074666519959767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,7,0.0846506655216217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,7,0.07051733136177063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,15,0.08360532919565837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,15,0.07052800059318542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,31,0.0844640036424001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,31,0.07036800185839336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,63,0.084906667470932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,63,0.07218666871388753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,127,0.0848426620165507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,127,0.07262933254241943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,255,0.0918293297290802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,255,0.07230400045712788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,511,0.1680906613667806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,511,0.1113973359266917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,7,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,15,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,255,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,255,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,511,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1023,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1023,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,2047,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,2047,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,4095,0.03760000069936117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,4095,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,8191,0.05979733169078827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,8191,0.03797333439191183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,16383,0.09940266609191895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,16383,0.05819733440876007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,32767,0.18134933710098267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,32767,0.09931199749310811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,65535,0.3449813524881999
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,65535,0.1802026629447937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,1,0.15689067045847574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,1,0.1343839963277181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,3,0.1566933294137319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,3,0.13436800241470337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,7,0.1567253371079763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,7,0.1342080036799113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,15,0.15657599767049155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,15,0.13429866234461466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,31,0.1563093364238739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,31,0.13473066687583923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,63,0.15667200088500977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,63,0.13524799545605978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,127,0.1564479966958364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,127,0.13586666186650595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,255,0.17081065972646078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,255,0.1339466671148936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,1,0.30163733164469403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,1,0.25703465938568115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,3,0.30002133051554364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,3,0.2570026715596517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,7,0.2998933394749959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,7,0.256661335627238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,15,0.30019734303156537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,15,0.25726934274037677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,31,0.30001600583394367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,31,0.25738133986790973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,63,0.3014613389968872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,63,0.26994667450586957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,127,0.29799999793370563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,127,0.2569493254025777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,3,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,3,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,7,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,7,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,15,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,31,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,31,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,63,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,127,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,127,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,255,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,255,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,511,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,511,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1023,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1023,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,2047,0.03832533210515976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,2047,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,4095,0.06020799775918325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,4095,0.03963200002908707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,8191,0.10212266445159912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,8191,0.06111466884613037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,16383,0.18345600366592407
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,16383,0.10146133104960124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,32767,0.35075199604034424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,32767,0.18440000216166177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,3,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,7,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,7,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,15,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,15,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,31,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,31,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,63,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,63,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,127,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,127,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,255,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,511,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,511,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1023,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1023,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,2047,0.0317493329445521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,2047,0.027530667682488758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,4095,0.0517493337392807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,4095,0.04404800136884054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,8191,0.08340266346931458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,8191,0.07227199772993724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,16383,0.14692266782124838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,16383,0.12594133615493774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,31,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,63,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,127,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,127,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,255,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,255,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,511,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1023,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1023,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,2047,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,2047,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,4095,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,4095,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,8191,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,8191,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,16383,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,16383,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,32767,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,32767,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,65535,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,65535,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,3,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,255,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1023,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,2047,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,2047,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,4095,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,4095,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,8191,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,8191,0.01741333305835724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,16383,0.021562665700912476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,16383,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,32767,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,32767,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,65535,0.04858666658401489
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,65535,0.04199466605981191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1,0.016506666938463848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,3,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,3,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,7,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,7,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,15,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,15,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,31,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,31,0.01681600014368693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,63,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,63,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,127,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,127,0.016314666718244553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,255,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,255,0.015754666179418564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,511,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,511,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1023,0.031370667119820915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1023,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,2047,0.05187733471393585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,2047,0.04621866842110952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,4095,0.08455999692281087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,4095,0.0745600014925003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,8191,0.1486186683177948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,8191,0.12798399726549783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,7,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,127,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,511,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,511,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1023,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1023,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,2047,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,2047,0.018160000443458557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,4095,0.01966933285196622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,4095,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,8191,0.022255999346574146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,8191,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,16383,0.03340800106525421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,16383,0.027119999130566914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,32767,0.04977599779764811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,32767,0.04121066629886627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,65535,0.07912000020345052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,65535,0.06823466718196869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,3,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,3,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,7,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,7,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,15,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,15,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,31,0.023024000227451324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,31,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,63,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,63,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,127,0.02149333308140437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,255,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,255,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,511,0.03262399882078171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,511,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1023,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1023,0.041946664452552795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,2047,0.08126933375994365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,2047,0.07227733234564464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,4095,0.1400266687075297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,4095,0.12232533097267151
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1,0.03566399961709976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1,0.03329066683848699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,3,0.035301332672437034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,3,0.03181866556406021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,7,0.033786666889985405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,7,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,15,0.033759998778502144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,15,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,31,0.03538133452335993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,31,0.03332799921433131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,63,0.035375999907652535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,63,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,127,0.0353973334034284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,127,0.033146666983763375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,255,0.0355679988861084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,255,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,511,0.052330667773882546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,511,0.045754666129748024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1023,0.0804799993832906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1023,0.07076799869537354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,1,0.05842666824658712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,1,0.054192001620928444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,3,0.058149332801500954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,3,0.05421333511670431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,7,0.05829866727193197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,7,0.053946668903032936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,15,0.05793599784374237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,15,0.05410666763782501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,31,0.058037335673967995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,31,0.05425600210825602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,63,0.05795200169086456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,63,0.05426133175690969
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,127,0.05916266640027364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,127,0.05415999889373779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,255,0.060165335734685264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,255,0.054133335749308266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,511,0.09089600046475728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,511,0.08083199958006541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,3,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,255,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,511,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1023,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1023,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,2047,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,2047,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,4095,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,4095,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,8191,0.03545066714286804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,8191,0.031167998909950256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,16383,0.05983466903368632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,16383,0.05230933427810669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,32767,0.10141866405804952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,32767,0.08896533648173015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,65535,0.1868106722831726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,65535,0.16076800227165222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,1,0.10537599523862202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,1,0.09682133793830872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,3,0.10543466607729594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,3,0.0971999963124593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,7,0.1074773371219635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,7,0.09748799602190654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,15,0.10679999987284343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,15,0.09723200400670369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,31,0.10731732845306396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,31,0.09702400366465251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,63,0.10543466607729594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,63,0.09683199723561604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,127,0.10582933823267619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,127,0.09745066364606221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,255,0.10558933019638062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,255,0.09682666261990865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,1,0.2019946575164795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,1,0.18544000387191772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,3,0.20164799690246582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,3,0.1856213410695394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,7,0.20190399885177612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,7,0.1857759952545166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,15,0.20162665843963623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,15,0.18525866667429605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,31,0.2016693353652954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,31,0.18693333864212036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,63,0.20311466852823892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,63,0.18685867389043173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,127,0.20173867543538412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,127,0.18710400660832724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,3,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,3,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,7,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,15,0.012047999848922094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,31,0.012453333785136541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,63,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,127,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,127,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,255,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,511,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,511,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1023,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1023,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,2047,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,2047,0.023562667270501454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,4095,0.0373279998699824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,4095,0.03246400008598963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,8191,0.062090665102005005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,8191,0.05154666801293691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,16383,0.10334933797518413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,16383,0.08948266506195068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,32767,0.19047466913859049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,32767,0.16275733709335327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,3,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,3,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,7,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,15,0.0141546664138635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,15,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,31,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,31,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,63,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,127,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,127,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,255,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,255,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,511,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,511,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1023,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1023,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,2047,0.050101334849993386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,2047,0.032469332218170166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,4095,0.08078399797280629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,4095,0.051498666405677795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,8191,0.143994669119517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,8191,0.08275733391443889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,16383,0.26792534192403156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,16383,0.14428800344467163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,3,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,15,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,15,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,31,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,63,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,127,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,127,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,255,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,511,0.011786667009194693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,511,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1023,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1023,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,2047,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,4095,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,4095,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,8191,0.017765333255132038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,8191,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,16383,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,16383,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,32767,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,32767,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,65535,0.046298667788505554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,3,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,65535,0.03181866556406021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,255,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,255,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,511,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1023,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,2047,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,4095,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,8191,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,16383,0.027386667827765148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,16383,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,32767,0.047135998805363975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,32767,0.0312266672650973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,65535,0.0766133318344752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,65535,0.04717866579691569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,3,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,3,0.01666133354107539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,7,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,7,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,15,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,15,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,31,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,31,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,63,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,63,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,127,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,127,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,255,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,255,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,511,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,511,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1023,0.04438933233420054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1023,0.029717333614826202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,2047,0.08080000181992848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,2047,0.05178666611512502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,4095,0.14412267009417215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,4095,0.0830080012480418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,7,0.07242133220036824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,8191,0.2709226608276367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,8191,0.14619732896486917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,15,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,255,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,511,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1023,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,2047,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,2047,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,4095,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,4095,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,8191,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,8191,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,16383,0.04868799944718679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,16383,0.03152533372243246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,32767,0.07866666714350383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,32767,0.049322664737701416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,65535,0.1365546683470408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,65535,0.0772159993648529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,3,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,3,0.025786665578683216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,7,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,7,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,15,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,15,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,31,0.024085332949956257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,31,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,63,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,63,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,127,0.023455999791622162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,127,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,255,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,255,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,511,0.044624000787734985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,511,0.03126933425664902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1023,0.07228800157705943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1023,0.04572799801826477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,2047,0.13570666313171387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,2047,0.0790719985961914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,4095,0.25282132625579834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,4095,0.13819199800491333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1,0.037461332976818085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,3,0.03772266705830892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,3,0.033728001018365227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,7,0.03745600084463755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,7,0.033600000043710075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,15,0.03774400055408478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,15,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,31,0.037690666814645134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,31,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,63,0.03750933210055033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,63,0.033861334125200905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,127,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,127,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,255,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,255,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,511,0.07286933561166127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,511,0.050250664353370667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1023,0.12786666552225748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1023,0.07670933504899342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,1,0.06630933284759521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,1,0.05633600056171417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,3,0.06650666892528534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,3,0.0563679983218511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,7,0.0665226678053538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,7,0.05596266686916351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,15,0.06634133557478587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,15,0.056287998954455055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,31,0.06636266907056172
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,31,0.05604266623655955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,63,0.06646933158238728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,63,0.056202664971351624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,127,0.06637866795063019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,127,0.058037335673967995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,255,0.07255466779073079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,255,0.05845333139101664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,511,0.1301866670449575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,511,0.0867733359336853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,511,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1023,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1023,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,2047,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,2047,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,4095,0.032826667030652366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,4095,0.023418667415777843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,8191,0.05600533386071523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,8191,0.036464000741640724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,16383,0.09340266386667888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,16383,0.05791999896367391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,32767,0.16819733381271362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,32767,0.0992746651172638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,65535,0.31824533144632977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,65535,0.17908799648284912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,1,0.1216159959634145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,1,0.10521599650382996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,3,0.12179733316103618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,3,0.10648000240325928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,7,0.12168533603350322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,7,0.10339200496673584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,15,0.12168000141779582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,15,0.10347200433413188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,31,0.1218826671441396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,63,0.12180266777674358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,31,0.10341866811116536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,63,0.1051626702149709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,127,0.12156266967455547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,127,0.10697066783905029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,255,0.13194666306177774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,255,0.10635200142860413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,1,0.22833067178726196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,1,0.19641067584355673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,3,0.2286293307940165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,3,0.19742933909098306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,7,0.2283680041631063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,7,0.19684267044067383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,15,0.229418675104777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,15,0.1973653237024943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,31,0.19615999857584634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,31,0.22971200942993164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,63,0.22825600703557333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,63,0.19699199994405112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,127,0.22686400016148886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,127,0.1978666583697001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1,0.012149333953857422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,3,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,3,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,7,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,7,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,15,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,31,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,31,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,63,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,127,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,127,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,255,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,511,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,511,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1023,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1023,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,2047,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,2047,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,4095,0.056330665946006775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,4095,0.03755733370780945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,8191,0.09531199932098389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,8191,0.06025599936644236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,16383,0.1731040080388387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,16383,0.10129066308339436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,32767,0.3304533362388611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,3,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,3,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,7,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,7,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,15,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,31,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,255,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,511,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,511,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1023,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1023,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,2047,0.0249439999461174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,2047,0.023434666295846302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,4095,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,4095,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,8191,0.06241600215435028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,8191,0.05460800230503082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,16383,0.10522133111953735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,16383,0.09086400270462036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,32767,0.1894773244857788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,32767,0.16357333461443582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,3,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,7,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,15,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1023,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,2047,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,2047,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,4095,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,4095,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,8191,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,8191,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,16383,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,16383,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,32767,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,32767,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,65535,0.024906667570273083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,65535,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,131071,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,131071,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,3,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,3,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,63,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,255,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,255,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,511,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,511,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1023,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1023,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,2047,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,4095,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,4095,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,8191,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,8191,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,16383,0.020288000504175823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,16383,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,32767,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,32767,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,65535,0.039477333426475525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,65535,0.030693332354227703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,131071,0.060085331400235496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,131071,0.05219733218352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,3,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,3,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,7,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,7,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,15,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,15,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,31,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,31,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,63,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,63,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,127,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,127,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,255,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,255,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,511,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,511,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1023,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1023,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,2047,0.04155199974775314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,2047,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,4095,0.062208001812299095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,4095,0.05384533107280731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,8191,0.10549867153167725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,8191,0.09105599919954936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,16383,0.19070400794347128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,16383,0.16313067078590393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,3,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,7,0.011946666985750198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,15,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,15,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,511,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,511,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1023,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1023,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,2047,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,2047,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,4095,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,4095,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,8191,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,8191,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,16383,0.024986666937669117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,16383,0.022970666488011677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,32767,0.04106133431196213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,32767,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,65535,0.06348266700903575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,65535,0.05410666763782501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,131071,0.10954667131106059
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,131071,0.0913813312848409
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,3,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,3,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,7,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,7,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,15,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,15,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,31,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,31,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,63,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,63,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,127,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,127,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,255,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,255,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,511,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,511,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1023,0.03568000098069509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1023,0.029477333029111225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,2047,0.05844266712665558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,2047,0.05175999800364176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,4095,0.09550399581591289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,4095,0.08469333251317342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,8191,0.17079466581344604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,8191,0.1478559970855713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1,0.027285332481066387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,3,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,3,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,7,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,7,0.02497066557407379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,15,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,15,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,31,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,31,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,63,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,63,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,127,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,127,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,255,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,255,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,511,0.03939733405907949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,511,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1023,0.05832533538341522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1023,0.05180266499519348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,2047,0.10107733805974324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,2047,0.08686400453249614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1,0.04148799926042557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,3,0.0415786678592364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,3,0.03930133332808813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,7,0.041797334949175514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,7,0.03939733405907949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,15,0.041562666495641075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,15,0.03944533318281174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,31,0.04155733436346054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,31,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,63,0.04156800111134847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,63,0.03934400031963984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,127,0.0414986660083135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,127,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,255,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,255,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,511,0.06452799836794536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,511,0.058304001887639366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1023,0.10123733679453532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1023,0.08884800473848979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,3,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,3,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,31,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,511,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1023,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1023,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,2047,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,2047,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,4095,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,4095,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,8191,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,8191,0.023077333966890972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,16383,0.039893334110577904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,16383,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,32767,0.06243200103441874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,32767,0.05327466626962026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,65535,0.1065013309319814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,65535,0.08867200215657552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,131071,0.19762667020161948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,131071,0.1628159979979197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,1,0.07432533303896587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,1,0.06859200199445088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,3,0.07445333401362102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,3,0.06806399921576183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,7,0.07432533303896587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,7,0.06843199829260509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,15,0.0746666689713796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,15,0.06829333305358887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,31,0.07461866736412048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,31,0.0685280015071233
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,63,0.07273600002129872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,63,0.06825600067774455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,127,0.0745066652695338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,127,0.06845333178838094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,255,0.0745600014925003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,255,0.06837333242098491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,511,0.11553600430488586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,511,0.10331733028093974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,1,0.13795199990272522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,1,0.1255413293838501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,3,0.13827733198801676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,3,0.12532266974449158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,7,0.13801599542299905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,7,0.1256533364454905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,15,0.13851733009020487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,15,0.12572800119717917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,31,0.13809600472450256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,31,0.12587733070055643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,63,0.1384213368097941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,63,0.12569600343704224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,127,0.1379146675268809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,127,0.12786133090655008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,255,0.1372213363647461
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,255,0.12562132875124613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1023,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,2047,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,2047,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,4095,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,4095,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,8191,0.039701332648595176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,8191,0.03162133445342382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,16383,0.060906668504079185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,16383,0.05402133365472158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,32767,0.10453866918881734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,32767,0.08944533268610637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,65535,0.1891040007273356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,65535,0.16056000192960104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,3,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,3,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,7,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,7,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,15,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,15,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,31,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,31,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,63,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,63,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,127,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,127,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,255,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,255,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,511,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,511,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1023,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1023,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,2047,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,2047,0.025455998877684276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,4095,0.060266668597857155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,4095,0.0397173340121905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,8191,0.1016533374786377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,8191,0.060677334666252136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,16383,0.18429332971572876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,16383,0.10153599580128987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,32767,0.35181868076324463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,32767,0.18443200985590616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,7,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,255,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1023,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1023,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,2047,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,2047,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,4095,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,4095,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,8191,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,8191,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,16383,0.019632000476121902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,16383,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,32767,0.02274133265018463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,32767,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,65535,0.038032000263532005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,65535,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,131071,0.059450666109720864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,131071,0.039733332892258964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,3,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,31,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,63,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,511,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,2047,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,2047,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,4095,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,4095,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,8191,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,8191,0.016719999412695568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,16383,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,16383,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,32767,0.03772799919048945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,32767,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,65535,0.058143998185793556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,65535,0.038058665891488395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,131071,0.09797867139180501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,131071,0.05945600072542826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,3,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,3,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,7,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,7,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,15,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,15,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,31,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,31,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,63,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,63,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,127,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,127,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,255,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,255,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,511,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,511,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1023,0.035301332672437034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1023,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,2047,0.060864001512527466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,2047,0.040106666584809623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,4095,0.10109333197275798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,4095,0.0617439995209376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,8191,0.18564265966415405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,8191,0.10327466328938802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,16383,0.35316268603007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,16383,0.18653867642084757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,3,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,3,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,255,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,511,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,511,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1023,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,2047,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,2047,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,4095,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,4095,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,8191,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,8191,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,16383,0.03912533322970072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,16383,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,32767,0.06005333364009857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,32767,0.03977599988381068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,65535,0.10086933771769206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,65535,0.061797335743904114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,131071,0.18781334161758423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,131071,0.10526933272679646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,3,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,3,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,7,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,7,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,15,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,15,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,31,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,31,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,63,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,63,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,127,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,127,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,255,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,255,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,511,0.03377600014209747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,511,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1023,0.05337599913279215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1023,0.035391998787721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,2047,0.09576533238093059
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,2047,0.05927466849486033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,4095,0.17220266660054526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,4095,0.09516266981760661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,8191,0.3260106643040975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,8191,0.16873067617416382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,3,0.029130667448043823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,3,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,7,0.029370665550231934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,7,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,15,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,15,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,31,0.029232000311215717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,31,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,63,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,63,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,127,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,127,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,255,0.03298133363326391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,255,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,511,0.05398400127887726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,511,0.03902933249870936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1023,0.09149866302808125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1023,0.056688000758488975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,2047,0.171999990940094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,2047,0.09726400176684062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1,0.045935998360315956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,3,0.045791998505592346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,3,0.03990400085846583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,7,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,7,0.03992533435424169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,15,0.04584000011285146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,15,0.04155199974775314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,31,0.047728002071380615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,31,0.04151466737190882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,63,0.04771199822425842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,63,0.041562666495641075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,127,0.04993600149949392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,127,0.04194133480389913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,255,0.05338666836420695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,255,0.04161066561937332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,511,0.09107733766237895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,511,0.062218666076660156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1023,0.16687466700871786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1023,0.0962720016638438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,7,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,31,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,511,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1023,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1023,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,2047,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,2047,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,4095,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,4095,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,8191,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,8191,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,16383,0.059706668059031166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,16383,0.03940266619126002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,32767,0.10238933563232422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,32767,0.06192000210285187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,65535,0.1853653391202291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,65535,0.10334400335947673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,131071,0.3533173402150472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,131071,0.18765334288279215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,1,0.08489599823951721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,1,0.070592001080513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,3,0.08452799916267395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,3,0.07242133220036824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,7,0.08505066235860188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,7,0.07220800220966339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,15,0.08481066425641377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,15,0.07256533205509186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,31,0.0846666693687439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,31,0.07200533151626587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,63,0.08463999629020691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,63,0.07073066631952922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,127,0.084714670976003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,127,0.07262933254241943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,255,0.0918293297290802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,255,0.07272533575693767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,511,0.16736533244450888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,511,0.11161067088445027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,1,0.1567573348681132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,1,0.1360053320725759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,3,0.15690132975578308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,3,0.13613866766293845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,7,0.15690666437149048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,7,0.1360479990641276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,15,0.15836800138155618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,15,0.13553067048390707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,31,0.15635200341542563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,31,0.13492266337076822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,63,0.15651733676592508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,63,0.1339466671148936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,127,0.15631999572118124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,127,0.1360266705354055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,255,0.17126933733622232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,255,0.13660266995429993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,15,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,15,0.011930666863918304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,31,0.011717333147923151
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,63,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,511,0.012522666404644648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1023,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1023,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,2047,0.020618667205174763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,2047,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,4095,0.037903999288876854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,4095,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,8191,0.059194669127464294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,8191,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,16383,0.10086933771769206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,16383,0.05868266522884369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,32767,0.18387200435002646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,32767,0.0997759997844696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,65535,0.35048532485961914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,65535,0.1789813240369161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,3,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,15,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,31,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,63,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,63,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,127,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,511,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,511,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1023,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1023,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,2047,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,2047,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,4095,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,4095,0.02293333411216736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,8191,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,8191,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,16383,0.061887999375661217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,16383,0.05418133238951365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,32767,0.10504532853762309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,32767,0.09091732899347942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,65535,0.18929600715637207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,65535,0.16265599926312765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,3,0.009103999783595404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,15,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,31,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,63,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,63,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,127,0.00890666681031386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,127,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,255,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,2047,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,2047,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,4095,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,8191,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,8191,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,16383,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,16383,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,32767,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,32767,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,65535,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,65535,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,131071,0.029877332349618275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,131071,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,3,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,15,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,31,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,31,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,63,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,63,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,127,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,255,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,511,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1023,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,2047,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,2047,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,4095,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,4095,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,8191,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,8191,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,16383,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,16383,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,32767,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,32767,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,65535,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,65535,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,131071,0.04171733558177948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,131071,0.03368533402681351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,3,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,63,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,255,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,255,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,511,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1023,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1023,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,2047,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,2047,0.02306666721900304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,4095,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,4095,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,8191,0.06237333516279856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,8191,0.05513600011666616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,16383,0.1050933301448822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,16383,0.09243733684221904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,32767,0.19150400161743164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,32767,0.16456533471743265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,15,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,127,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,511,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1023,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1023,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,2047,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,4095,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,8191,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,8191,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,16383,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,16383,0.020346666375796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,32767,0.025626666843891144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,32767,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,65535,0.04186666508515676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,65535,0.03145066648721695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,131071,0.06308266520500183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,131071,0.05383466680844625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,3,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,3,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,7,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,7,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,15,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,15,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,31,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,31,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,63,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,63,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,127,0.015637333194414776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,127,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,255,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,255,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,511,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,511,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1023,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1023,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,2047,0.0399893323580424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,2047,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,4095,0.06238933404286703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,4095,0.056090667843818665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,8191,0.10603200395901997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,8191,0.09092266360918681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,16383,0.19115199645360312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,16383,0.1646773318449656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,3,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,3,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,7,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,7,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,15,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,15,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,31,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,31,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,63,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,63,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,127,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,127,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,255,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,255,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,511,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,511,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1023,0.03568000098069509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1023,0.029418667157491047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,2047,0.0599839985370636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,2047,0.05248000224431356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,4095,0.09712533156077068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,4095,0.08501332998275757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,3,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,3,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,7,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,7,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,15,0.027109332382678986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,15,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,31,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,31,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,63,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,63,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,127,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,127,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,255,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,255,0.02476799984773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,511,0.03872533390919367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,511,0.03355200091997782
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1023,0.058186665177345276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1023,0.05217599868774414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,2047,0.10089600086212158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,2047,0.08702400326728821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,15,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,63,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,255,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,511,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1023,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1023,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,2047,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,2047,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,4095,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,4095,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,8191,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,8191,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,16383,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,16383,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,32767,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,32767,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,65535,0.06237333516279856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,65535,0.054154664278030396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,131071,0.1111306647459666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,131071,0.09119466940561931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1,0.041738669077555336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,3,0.04205866654713949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,3,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,7,0.04177066683769226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,7,0.03990933299064636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,15,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,15,0.03945599993069967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,31,0.04166933397452036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,31,0.039461334546407066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,63,0.041536000867684685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,63,0.03945599993069967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,127,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,127,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,255,0.04402133325735728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,255,0.03842666745185852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,511,0.06451733410358429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,511,0.05799466868241628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1023,0.10139200091362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1023,0.08888000249862671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,1,0.07260799904664357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,1,0.06801066795984904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,3,0.07419200241565704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,3,0.06851199766000111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,7,0.07433600227038066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,7,0.0683840016523997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,15,0.07459199925263722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,15,0.06820799907048543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,31,0.0743093341588974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,31,0.06831466654936473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,63,0.07438399891058604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,63,0.06824000179767609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,127,0.07442133128643036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,127,0.06674133241176605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,255,0.07449066638946533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,255,0.06850133339564006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,511,0.11541333794593811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,511,0.10309333602587382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,511,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1023,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,2047,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,2047,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,4095,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,4095,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,8191,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,8191,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,16383,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,16383,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,32767,0.06397866706053416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,32767,0.05423999826113383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,65535,0.10706133643786113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,65535,0.08872000376383464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,131071,0.19785600900650024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,131071,0.1623199979464213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,3,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,15,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,31,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,511,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1023,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,2047,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,2047,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,4095,0.03939199944337209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,4095,0.024117333193620045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,8191,0.05938666562239329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,8191,0.03939199944337209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,16383,0.10007466872533162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,16383,0.059717332323392235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,32767,0.18187199036280313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,32767,0.10030399759610494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,65535,0.344709316889445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,65535,0.18000000715255737
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,31,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,63,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1023,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,2047,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,2047,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,4095,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,4095,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,8191,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,8191,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,16383,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,16383,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,32767,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,32767,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,65535,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,65535,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,131071,0.04257600009441376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,131071,0.029487999776999157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,31,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,2047,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,2047,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,4095,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,4095,0.015962666521469753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,8191,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,8191,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,16383,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,16383,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,32767,0.022874665757020313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,32767,0.020975999534130096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,65535,0.039034667114416756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,65535,0.025424001117547352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,131071,0.0605973352988561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,131071,0.04185600082079569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,3,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,3,0.012565333396196365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,7,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,7,0.012389333297808966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,15,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,15,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,31,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,31,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,63,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,63,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,127,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,127,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,255,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,255,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,511,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1023,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1023,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,2047,0.03980266551176707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,2047,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,4095,0.06112533311049143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,4095,0.0408693328499794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,8191,0.10160533587137859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,8191,0.06241600215435028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,16383,0.18535999457041422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,16383,0.10307199756304423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,32767,0.34935466448465985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,32767,0.185808002948761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,7,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,7,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,15,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,15,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,31,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,31,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,127,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,255,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,511,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1023,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1023,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,2047,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,2047,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,4095,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,4095,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,8191,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,8191,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,16383,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,16383,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,32767,0.039919999738534294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,32767,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,65535,0.059936001896858215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,65535,0.04090133309364319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,131071,0.10087999701499939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,131071,0.06326933205127716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,3,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,3,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,7,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,7,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,15,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,15,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,31,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,31,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,63,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,63,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,127,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,127,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,255,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,255,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,511,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,511,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1023,0.033376000821590424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1023,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,2047,0.06058133145173391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,2047,0.04148799926042557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,4095,0.10292266805966695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,4095,0.061946665247281395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,8191,0.18474666277567545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,8191,0.10318400462468465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,16383,0.3503520091374715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,16383,0.18789867560068765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,3,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,3,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,7,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,7,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,15,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,15,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,31,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,31,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,63,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,63,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,127,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,127,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,255,0.01974933346112569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,255,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,511,0.03321066747109095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,511,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1023,0.052058666944503784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1023,0.03524799893299738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,2047,0.09563199679056804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,2047,0.058186665177345276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,4095,0.1718239982922872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,4095,0.09641066193580627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,3,0.02921066681543986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,3,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,7,0.029157333076000214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,7,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,15,0.02924799919128418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,15,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,31,0.028725333511829376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,31,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,63,0.029258665939172108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,63,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,127,0.028704000016053517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,127,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,255,0.0320266659061114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,255,0.02651199946800868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,511,0.05390933156013489
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,511,0.03733866661787033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1023,0.09123200178146362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1023,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,2047,0.1723626653353373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,2047,0.09776533643404643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,3,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,127,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,511,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,2047,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,4095,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,4095,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,8191,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,8191,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,16383,0.037962667644023895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,16383,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,32767,0.05905066430568695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,32767,0.039408000806967415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,65535,0.10127466917037964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,65535,0.0621013343334198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,131071,0.18525866667429605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,131071,0.10499733686447144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1,0.04621333380540212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,3,0.047685335079828896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,3,0.03990933299064636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,7,0.04734399914741516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,7,0.0397119993964831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,15,0.04735999802748362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,15,0.041802664597829185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,31,0.047797332207361855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,31,0.04155733436346054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,63,0.04780266682306925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,63,0.041536000867684685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,127,0.04921066761016846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,127,0.04168533285458883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,255,0.053130666414896645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,255,0.041690667470296226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,511,0.09121599793434143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,511,0.062122667829195656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1023,0.16525333126386008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1023,0.0971999963124593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,1,0.08477866649627686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,1,0.07036800185839336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,3,0.08495466907819112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,3,0.07233066856861115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,7,0.08482133348782857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,7,0.07249066730340321
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,15,0.08462933699289958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,8191,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,15,0.07070933282375336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,31,0.08489066362380981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,31,0.07061333457628886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,63,0.08475200335184734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,63,0.07047466437021892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,127,0.08505599697430928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,127,0.07272000114123027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,255,0.09291199843088786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,255,0.0738506664832433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,511,0.16681599617004395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,511,0.11146133144696553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,511,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1023,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1023,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,2047,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,15,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,2047,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,4095,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,4095,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,8191,0.038218667109807335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,8191,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,16383,0.06015466650327047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,16383,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,32767,0.10021332899729411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,32767,0.06074133515357971
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,65535,0.1839946707089742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,65535,0.10126933455467224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,131071,0.3506506681442261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,131071,0.1872373421986898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,7,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,511,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1023,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,2047,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,4095,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,4095,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,8191,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,8191,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,16383,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,16383,0.03947199881076813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,32767,0.05239999790986379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,32767,0.06440000236034393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,65535,0.08961600065231323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,65535,0.10748799641927083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,131071,0.1632213294506073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,131071,0.19956799348195395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,63,0.010175999874869982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,127,0.010378666842977205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,511,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,2047,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,2047,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,4095,0.011871999750534693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,4095,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,8191,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,8191,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,16383,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,16383,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,32767,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,32767,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,65535,0.023872000475724537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,65535,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,131071,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,131071,0.02738133321205775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,7,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,15,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,63,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,127,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,511,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1023,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,2047,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,2047,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,4095,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,4095,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,8191,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,8191,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,16383,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,16383,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,32767,0.020773333807786305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,32767,0.02024000013868014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,65535,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,65535,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,131071,0.029525332152843475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,7,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,131071,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,511,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1023,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,2047,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,2047,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,4095,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,4095,0.022005334496498108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,8191,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,8191,0.03141333411137263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,16383,0.06195733447869619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,16383,0.054058666030565895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,32767,0.09116266171137492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,32767,0.10376532872517903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,65535,0.1630400021870931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,65535,0.19040000438690186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,7,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,31,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,511,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1023,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,2047,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,2047,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,4095,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,4095,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,8191,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,8191,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,16383,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,16383,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,32767,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,32767,0.020853333175182343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,65535,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,131071,0.04068800061941147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,131071,0.035760000348091125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,32767,0.1835413376490275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,3,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,7,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,7,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,15,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,15,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,31,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,127,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,127,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,255,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,511,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,511,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1023,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1023,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,2047,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,2047,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,4095,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,4095,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,8191,0.06301866471767426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,8191,0.05415999889373779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,16383,0.10566932956377666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,16383,0.09110400080680847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,32767,0.19350399573644003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,32767,0.1630346675713857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,3,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,3,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,7,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,7,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,15,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,15,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,31,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,31,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,63,0.013925333817799887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,63,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,127,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,127,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,255,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,255,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,511,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,511,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1023,0.023354666928450268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1023,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,2047,0.04053866614898046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,2047,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,4095,0.06345066428184509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,4095,0.056320001681645714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,8191,0.10571733117103577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,8191,0.09322667121887207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1,0.017968000223239262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,3,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,3,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,7,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,7,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,15,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,15,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,31,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,31,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,63,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,63,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,127,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,127,0.017583999782800674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,255,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,255,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,511,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1023,0.03566933423280716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1023,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,2047,0.05971199770768484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,2047,0.053802669048309326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,4095,0.09803199768066406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,4095,0.08489066362380981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,7,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,63,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,255,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1023,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1023,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,2047,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,2047,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,4095,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,4095,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,8191,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,8191,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,16383,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,16383,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,32767,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,32767,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,65535,0.04190400242805481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,65535,0.031856000423431396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,131071,0.053599998354911804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1,0.0271519993742307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,3,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,3,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,7,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,7,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,15,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,15,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,31,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,31,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,63,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,63,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,127,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,127,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,255,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,255,0.024373332659403484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,511,0.03941866755485535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,511,0.033626665671666466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1023,0.0583840012550354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1023,0.05179200073083242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,2047,0.1009226640065511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,2047,0.08692266543706258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1,0.03868799904982249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,3,0.04156800111134847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,3,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,7,0.04155733436346054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,7,0.039749334255854286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,15,0.04153066625197729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,15,0.039706667264302574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,31,0.042319998145103455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,31,0.039493332306543984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,63,0.04192533095677694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,63,0.03909866760174433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,127,0.04144000013669332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,127,0.03849066545565923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,255,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,255,0.037845333417256675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,511,0.0641546646753947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,511,0.05798399945100149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1023,0.10132267077763875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1023,0.08893866340319316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,31,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,63,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,511,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1023,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,2047,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,2047,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,4095,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,4095,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,8191,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,8191,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,16383,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,16383,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,32767,0.04147200038035711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,32767,0.03181333343187968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,65535,0.06324266890684764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,65535,0.05310933291912079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,131071,0.10942400495211284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,131071,0.09145067135492961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,3,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,7,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,511,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1023,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,2047,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,2047,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,4095,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,4095,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,8191,0.03839466720819473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,8191,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,16383,0.060165335734685264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,16383,0.039834665755430855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,32767,0.10177066922187805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,32767,0.06195733447869619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,65535,0.1833440065383911
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,65535,0.10316266616185506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,131071,0.3513013521830241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,131071,0.18755733966827393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,63,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,2047,0.011648000528415045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,2047,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,4095,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,4095,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,8191,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,8191,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,16383,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,16383,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,32767,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,32767,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,65535,0.02629866699377696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,65535,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,131071,0.03172266731659571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,131071,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,255,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,511,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,511,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1023,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1023,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,2047,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,2047,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,4095,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,8191,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,8191,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,16383,0.020047999918460846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,16383,0.01958400011062622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,32767,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,32767,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,65535,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,65535,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,131071,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,131071,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,127,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,511,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1023,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1023,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,2047,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,2047,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,4095,0.037632000943024956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,4095,0.024069334069887798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,8191,0.06035199761390686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,8191,0.03956266740957896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,16383,0.10040000081062317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,16383,0.060005332032839455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,32767,0.18211734294891357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,32767,0.09982400139172871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,65535,0.34618667761484784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,65535,0.1808533271153768
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,15,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,255,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,511,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,511,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1023,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,2047,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,2047,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,4095,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,4095,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,8191,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,8191,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,16383,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,16383,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,32767,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,32767,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,65535,0.04161066561937332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,65535,0.026911998788515728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,131071,0.06284800171852112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,131071,0.043738668163617454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,3,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,3,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,7,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,7,0.012261333564917246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,15,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,15,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,31,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,31,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,63,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,127,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,127,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,255,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,255,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,511,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,511,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1023,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1023,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,2047,0.038719999293486275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,2047,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,4095,0.06132799883683523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,4095,0.04133866727352142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,8191,0.10274666547775269
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,8191,0.062463998794555664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,16383,0.18498132626215616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,16383,0.10406399766604106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,32767,0.3503466844558716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,32767,0.18592000007629395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,3,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,3,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,7,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,7,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,15,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,15,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,31,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,31,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,63,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,63,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,127,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,127,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,255,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,255,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,511,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,511,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1023,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1023,0.023386667172114056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,2047,0.0620959997177124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,2047,0.04131199916203817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,4095,0.1032533347606659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,4095,0.06189866860707601
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,8191,0.18579200903574625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,8191,0.10436800122261047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,3,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,3,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,7,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,7,0.019685332973798115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,15,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,15,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,31,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,31,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,63,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,63,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,127,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,127,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,255,0.019658666104078293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,255,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,511,0.03313600023587545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,511,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1023,0.052517334620157875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1023,0.034287999073664345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,2047,0.09717333316802979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,2047,0.058117335041364036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,4095,0.17279465993245444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,4095,0.09591466188430786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,3,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,3,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,31,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,255,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,511,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1023,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,2047,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,2047,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,4095,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,4095,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,8191,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,8191,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,16383,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,16383,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,32767,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,32767,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,65535,0.060191998879114784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,65535,0.04165333261092504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,131071,0.10238400101661682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,131071,0.06282133360703786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1,0.02920000006755193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,3,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,3,0.025413334369659424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,7,0.027994667490323383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,7,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,15,0.028330666323502857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,15,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,31,0.028912000358104706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,31,0.025455998877684276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,63,0.02863466739654541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,63,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,127,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,127,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,255,0.03339199970165888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,255,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,511,0.05357333521048228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,511,0.03753600021203359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1023,0.09092266360918681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1023,0.05628266433874766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,2047,0.17434666554133096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,2047,0.09887466828028361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1,0.04783466458320618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1,0.0413973331451416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,3,0.04773333172003428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,3,0.041834667325019836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,7,0.04775466521581014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,7,0.03986666599909464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,15,0.04587199787298838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,15,0.03991466760635376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,31,0.04602666695912679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,31,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,63,0.04621866842110952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,63,0.03977599988381068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,127,0.04816000163555145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,127,0.04195199906826019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,255,0.052576000491778054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,255,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,511,0.0918346643447876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,511,0.062165334820747375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1023,0.16660799582799277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1023,0.09702933828035991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,7,0.012058666596810022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,15,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,15,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,31,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,63,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,511,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1023,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,2047,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,2047,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,4095,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,4095,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,8191,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,8191,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,16383,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,16383,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,32767,0.06047999858856201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,32767,0.04098666707674662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,65535,0.10100799798965454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,65535,0.059952000776926674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,131071,0.18564265966415405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,131071,0.10552000006039937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,511,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,2047,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,2047,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,4095,0.015935999651749928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,4095,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,8191,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,8191,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,16383,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,16383,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,32767,0.041663999358812966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,32767,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,65535,0.0643039991458257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,65535,0.05385066568851471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,131071,0.10983467102050781
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,131071,0.09171199798583984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,63,0.010330666477481524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,127,0.010079999764760336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1023,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,2047,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,2047,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,4095,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,4095,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,8191,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,8191,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,16383,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,16383,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,32767,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,32767,0.02741866558790207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,65535,0.03146666785081228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,65535,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,131071,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,131071,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,3,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,7,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,7,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,15,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,31,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,31,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,63,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,2047,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,2047,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,4095,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,4095,0.012272000312805176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,8191,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,8191,0.016757333030303318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,16383,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,16383,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,32767,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,32767,0.023418667415777843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,65535,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,65535,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,131071,0.029765332738558452
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,131071,0.027647999425729115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,15,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,63,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,511,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1023,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1023,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,2047,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,2047,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,4095,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,4095,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,8191,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,8191,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,16383,0.0414986660083135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,16383,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,32767,0.06234666705131531
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,32767,0.05385066568851471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,65535,0.10847999652226765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,65535,0.09116799632708232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,131071,0.19915199279785156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,131071,0.1630506714185079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,7,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,15,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,511,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,2047,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,4095,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,4095,0.013855999956528345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,8191,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,8191,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,16383,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,16383,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,32767,0.022469334304332733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,32767,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,65535,0.023376000424226124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,65535,0.02306666721900304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,131071,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,131071,0.029733332494894665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1,0.012485332787036896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,3,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,7,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,15,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,255,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,511,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,2047,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,2047,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,4095,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,4095,0.022426667312781017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,8191,0.04012266546487808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,8191,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,16383,0.062352001667022705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,16383,0.054698665936787925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,32767,0.10421866178512573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,32767,0.09073066711425781
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,65535,0.1895093321800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,65535,0.16299733519554138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,3,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,7,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,7,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,15,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,15,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,31,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,63,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,127,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,255,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,255,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,511,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,511,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1023,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1023,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,2047,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,2047,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,4095,0.041850666205088295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,4095,0.03197333216667175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,8191,0.06259733438491821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,8191,0.055530667304992676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,16383,0.1074666678905487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,16383,0.09244799613952637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,3,0.014511999984582266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,3,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,7,0.014565333724021912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,7,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,15,0.014490666488806406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,15,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,31,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,31,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,63,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,63,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,127,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,127,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,255,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,255,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,511,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,511,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1023,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1023,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,2047,0.041152000427246094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,2047,0.03322133421897888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,4095,0.06308799982070923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,4095,0.05611733098824819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,8191,0.1056106686592102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,8191,0.09300800164540608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,255,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,511,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1023,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1023,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,2047,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,2047,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,4095,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,4095,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,8191,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,8191,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,16383,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,16383,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,32767,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,32767,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,65535,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,65535,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,131071,0.04494933287302653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,131071,0.035786665976047516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,3,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,3,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,7,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,7,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,15,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,15,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,31,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,31,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,63,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,63,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,127,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,127,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,255,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,255,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,511,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,511,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1023,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1023,0.031311998764673867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,2047,0.059989333152770996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,2047,0.0525439977645874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,4095,0.09806399544080098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,4095,0.08513599634170532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,3,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,3,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,7,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,7,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,15,0.02661866694688797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,15,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,31,0.026144000391165417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,31,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,63,0.02609066665172577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,63,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,127,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,127,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,255,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,255,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,511,0.038592000802357994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,511,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1023,0.058304001887639366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1023,0.052298665046691895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,2047,0.10058666268984477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,2047,0.08747733632723491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,127,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1023,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,2047,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,2047,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,4095,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,4095,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,8191,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,8191,0.018090666582187016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,16383,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,16383,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,32767,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,32767,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,65535,0.04016000032424927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,65535,0.032618666688601174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,131071,0.06443200012048085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,131071,0.05439466734727224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,3,0.012661332885424295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,7,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,15,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,31,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,31,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,63,0.01240533341964086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,255,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1023,0.01250133290886879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,2047,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,2047,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,4095,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,4095,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,8191,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,8191,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,16383,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,16383,0.025072000920772552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,32767,0.06072533130645752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,32767,0.03990933299064636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,65535,0.10177066922187805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,65535,0.06180266539255778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,131071,0.18739734093348184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,131071,0.10562666257222493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,3,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,7,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,15,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,15,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,511,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,2047,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,2047,0.012245333443085352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,4095,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,4095,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,8191,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,8191,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,16383,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,16383,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,32767,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,32767,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,65535,0.031210665901501972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,65535,0.029989334444204967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,131071,0.035760000348091125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,131071,0.035930665830771126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,511,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1023,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1023,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,2047,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,2047,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,4095,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,4095,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,8191,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,8191,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,16383,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,16383,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,32767,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,32767,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,65535,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,65535,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,131071,0.03305066625277201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,131071,0.029546665648619335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,7,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,15,0.010464000205198923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,127,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,511,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,511,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1023,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1023,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,2047,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,2047,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,4095,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,4095,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,8191,0.03827733298142751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,8191,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,16383,0.060415998101234436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,16383,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,32767,0.10197866956392924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,32767,0.06136533121267954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,65535,0.18398932615915933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,65535,0.1036906639734904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,131071,0.35177067915598553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,131071,0.1883946657180786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,7,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,511,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1023,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1023,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,2047,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,2047,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,4095,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,4095,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,8191,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,8191,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,16383,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,16383,0.020981334149837494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,32767,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,32767,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,65535,0.028207999964555103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,65535,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,131071,0.048469334840774536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,131071,0.0329066663980484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,15,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,511,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,511,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1023,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1023,0.014682666709025701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,2047,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,2047,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,4095,0.038848000268141426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,4095,0.025626666843891144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,8191,0.040005333721637726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,8191,0.06027733286221822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,16383,0.06018133461475372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,16383,0.10148266951243083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,32767,0.10083733002344768
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,32767,0.18331199884414673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,65535,0.3461546500523885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,3,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,3,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,7,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,65535,0.18182400862375894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,7,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,15,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,15,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,31,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,31,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,63,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,127,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,127,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,255,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,511,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,511,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1023,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1023,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,2047,0.03944533318281174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,2047,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,4095,0.06158933540185293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,4095,0.041984001795450844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,8191,0.06316266457239787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,8191,0.10357333223025005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,16383,0.1864853302637736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,16383,0.10427199800809224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,3,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,3,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,7,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,7,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,15,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,15,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,31,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,31,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,63,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,63,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,127,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,127,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,255,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,255,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,511,0.019466667125622433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,511,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1023,0.03549866626660029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1023,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,2047,0.06221333146095276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,2047,0.04181866844495138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,4095,0.10519466797510783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,4095,0.06251733501752217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,8191,0.18851200739542642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,8191,0.10524266958236694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,15,0.010384000216921171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,63,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1023,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,2047,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,2047,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,4095,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,4095,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,8191,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,8191,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,16383,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,16383,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,32767,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,32767,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,65535,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,65535,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,131071,0.06532266736030579
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,131071,0.04400533437728882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,3,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,3,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,7,0.020693333198626835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,7,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,15,0.01977066695690155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,15,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,31,0.020586666961510975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,31,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,63,0.020394666741291683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,63,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,65535,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,127,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,127,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,255,0.019509332875410717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,255,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,511,0.033573334415753685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,511,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1023,0.05397866666316986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1023,0.03570133447647095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,2047,0.09834133585294087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,2047,0.0582239975531896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1,0.02880000074704488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,4095,0.09713066617647807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,4095,0.175818661848704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,3,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,3,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,7,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,7,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,15,0.029274667302767437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,15,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,31,0.02942933390537898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,31,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,63,0.029450667401154835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,63,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,127,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,127,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,255,0.033226666351159416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,255,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,511,0.054005334774653115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,511,0.03814399987459183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1023,0.09101333220799764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1023,0.056176001826922096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,2047,0.17500799894332886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,2047,0.09922666351000468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,511,0.012037333101034164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,511,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1023,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,2047,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,2047,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,4095,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,4095,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,8191,0.020261333634455998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,8191,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,16383,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,16383,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,32767,0.0408693328499794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,32767,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,65535,0.062074666221936546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,65535,0.04147200038035711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,131071,0.10267200072606404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,131071,0.06243733565012614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1,0.048207998275756836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1,0.04005333284536997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1,0.04822933177153269
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1,0.03984533250331879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,3,0.04795200129350027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,3,0.048138668139775596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,3,0.03979199876387914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,3,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,7,0.04827733337879181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,7,0.049360002080599465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,7,0.041936000188191734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,7,0.04167999823888143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,15,0.04984533290068308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,15,0.050101334849993386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,15,0.043696001172065735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,15,0.04345066845417023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,31,0.06018666426340739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,31,0.060138667623202004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,31,0.052245333790779114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,31,0.05379733443260193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,63,0.05994133154551188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,63,0.05983999868233999
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,63,0.05388266841570536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,63,0.05410666763782501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,127,0.060234665870666504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,127,0.060309335589408875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,127,0.05366933345794678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,127,0.05407466491063436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,255,0.06043200194835663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,255,0.07045333087444305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,255,0.05393599967161814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,255,0.06409599880377452
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,511,0.06028800209363302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,511,0.1034346620241801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,511,0.054144000013669334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,511,0.09723732868830363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1023,0.06158933540185293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1023,0.16910399993260702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1023,0.054101333022117615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1023,0.16263999541600546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,2047,0.06028800209363302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,2047,0.3039413293202718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,2047,0.053930665055910744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,2047,0.29612799485524494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,4095,0.06051200131575266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,4095,0.5707733233769735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,4095,0.05397333204746246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1,0.04784533381462097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,131071,0.0645546664794286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1,0.04799999793370565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1,0.039919999738534294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,4095,0.5631466706593832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,3,0.0481279989083608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,3,0.04870399832725525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,3,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1,0.04014399896065394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,3,0.03990933299064636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,7,0.049178664882977806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,7,0.041797334949175514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,7,0.049957334995269775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,7,0.04167999823888143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,15,0.050197333097457886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,15,0.04993600149949392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,15,0.04307200014591217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,31,0.06009600063165029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,31,0.060005332032839455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,31,0.05392533540725708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,31,0.05376533170541128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,63,0.06036800146102905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,63,0.0602400004863739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,63,0.05381333331267039
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,63,0.0539626677831014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,127,0.0602400004863739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,127,0.060533334811528526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,127,0.05392533540725708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,127,0.05403733253479004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,255,0.06026133398214976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,255,0.07044266661008199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,255,0.05372266471385956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,255,0.06449066599210103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,511,0.060229331254959106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,511,0.10336533188819885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,511,0.05376000205675761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,511,0.09732266267140706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1023,0.060421332716941833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1023,0.05387733379999796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1023,0.16909333070119223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,2047,0.06163733204205831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1023,0.16289066274960837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,2047,0.05429866909980774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,2047,0.3040693402290344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,2047,0.2959360082944234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,4095,0.061103999614715576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,4095,0.05398933092753092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,4095,0.5724586645762125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1,0.0479360024134318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,4095,0.5662933190663656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1,0.0479360024134318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1,0.0401653324564298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1,0.04090133309364319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,3,0.04786666731039683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,3,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,3,0.03987200061480204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,3,0.04297066728274027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,7,0.0498933345079422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,7,0.0521066685517629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,7,0.041738669077555336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,7,0.04371733466784159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,15,0.05037866532802582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,15,0.05014933149019877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,15,0.04359466830889384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,15,0.04350399971008301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,31,0.05991466840108236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,31,0.05994666616121928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,31,0.05216533442338308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,31,0.05217599868774414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,63,0.06018133461475372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,63,0.06001600126425425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,63,0.0539680023988088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,63,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,127,0.06011733412742615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,127,0.060309335589408875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,127,0.05409066875775655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,127,0.054048001766204834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,255,0.06053866446018219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,255,0.07047466437021892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,255,0.05397866666316986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,255,0.06419733166694641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,511,0.06033066908518473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,511,0.10354133447011311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,511,0.054133335749308266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,511,0.09722666939099629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1023,0.06178666651248932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1023,0.17073599497477213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1023,0.05422399938106537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1023,0.16485866904258728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,2047,0.06132266422112783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,2047,0.30450133482615155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,2047,0.05418133238951365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,2047,0.2960960070292155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,4095,0.06132266422112783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,4095,0.5791786511739095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,4095,0.05420266588528951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,4095,0.5620533227920532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,3,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,3,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,3,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,3,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,7,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,7,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,7,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,7,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,15,0.01392000044385592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,15,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,15,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,31,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,31,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,31,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,31,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,63,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,63,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,63,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,63,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,127,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,127,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,127,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,255,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,255,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,255,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,255,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,511,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,511,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,511,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,511,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1023,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1023,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1023,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,2047,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,2047,0.027877333263556164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,2047,0.015834666788578033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,2047,0.025829332570234936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,4095,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,4095,0.043696001172065735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,4095,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,4095,0.03751999884843826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1,0.009354666496316591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,3,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,3,0.009535999968647957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,3,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,7,0.009296000003814697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,7,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,7,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,15,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,15,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,31,0.009194666519761086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,31,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,63,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,63,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,127,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,127,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,511,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,511,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,511,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1023,0.020021333048741024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1023,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1023,0.025455998877684276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1023,0.030117332935333252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,2047,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,2047,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,2047,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,2047,0.034048000971476235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,4095,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,4095,0.04382933179537455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,4095,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,4095,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,8191,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,8191,0.06644266843795776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,8191,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,8191,0.06247999767462412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,16383,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,16383,0.11222400267918904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,16383,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,16383,0.10141332944234212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1,0.009306666751702627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,3,0.009328000247478485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,3,0.009050666665037474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,7,0.00943999985853831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,7,0.009338666374484697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,15,0.009381333366036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,15,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,31,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,127,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,255,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,511,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,511,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,511,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1023,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1023,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1023,0.024319998919963837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1023,0.02938133229811986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,2047,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,2047,0.030991998811562855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,2047,0.02492800106604894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,2047,0.03332266708215078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,4095,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,4095,0.04278933505217234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,4095,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,4095,0.043285335103670754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,8191,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,8191,0.06610666712125142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,8191,0.025029333929220837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,8191,0.06237333516279856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,16383,0.020879998803138733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,16383,0.11153067151705424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,16383,0.023573334018389385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,16383,0.10123200217882793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1,0.009018666421373686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,3,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,3,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,3,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,7,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,7,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,7,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,15,0.009861333295702934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,15,0.009712000067035357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,31,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,31,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,63,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,127,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,127,0.013936000565687815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,255,0.013973332941532135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,255,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,511,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,511,0.02170666555563609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,511,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,511,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1023,0.02085866779088974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1023,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1023,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1023,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,2047,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,2047,0.03154666721820831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,2047,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,2047,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,4095,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,4095,0.04324266811211904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,4095,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,4095,0.04348266621430715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,8191,0.019850666324297588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,8191,0.06651199857393901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,8191,0.02473066747188568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,8191,0.06267199913660686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,16383,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,16383,0.11153067151705424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,16383,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,16383,0.10143466790517171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,15,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,31,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,127,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,511,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1023,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1023,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1023,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,2047,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,2047,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,2047,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,4095,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,4095,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,4095,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,4095,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,8191,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,8191,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,8191,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,8191,0.01670933390657107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,16383,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,16383,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,16383,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,16383,0.017797333498795826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,3,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,3,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,3,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,7,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,7,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,7,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,15,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,15,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,15,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,31,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,31,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,31,0.012223999947309494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,31,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,63,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,63,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,63,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,127,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,127,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,127,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,127,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,255,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,255,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,255,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,255,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,511,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,511,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,511,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,511,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1023,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1023,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1023,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,2047,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,2047,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,2047,0.029487999776999157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,4095,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,4095,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,4095,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,4095,0.05175999800364176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,8191,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,8191,0.09731733798980713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,8191,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,8191,0.0909546713034312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,16383,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,16383,0.18120533227920532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,16383,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,16383,0.17299199104309082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,3,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,3,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,3,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,7,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,7,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,7,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,15,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,15,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,15,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,15,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,31,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,31,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,63,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,63,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,63,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,63,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,127,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,127,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,127,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,255,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,255,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,511,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,511,0.01833600054184596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,511,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,511,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1023,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1023,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1023,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1023,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,2047,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,2047,0.0352906659245491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,2047,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,2047,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,4095,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,4095,0.055813332398732506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,4095,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,4095,0.051674668987592064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,8191,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,8191,0.0969493289788564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,8191,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,8191,0.09087999661763509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,16383,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,16383,0.181002676486969
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,16383,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,16383,0.1727786660194397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1,0.014666666587193808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,3,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,3,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,7,0.015743999431530636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,7,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,15,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,15,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,31,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,31,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,31,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,63,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,63,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,63,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,63,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,127,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,127,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,127,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,255,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,255,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,255,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,511,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,511,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,511,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1023,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1023,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1023,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,2047,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,2047,0.03498133271932602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,2047,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,2047,0.03154666721820831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,4095,0.016810666769742966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,4095,0.05579733351866404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,4095,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,4095,0.05031466484069824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,8191,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,8191,0.09723732868830363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,8191,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,8191,0.0909440020720164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,16383,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,16383,0.18088533480962118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,16383,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,16383,0.17297067244847616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,3,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,15,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,127,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,511,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,511,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1023,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1023,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,2047,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,2047,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,2047,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,4095,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,4095,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,4095,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,4095,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,8191,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,8191,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,8191,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,8191,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,16383,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,16383,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,16383,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,16383,0.019498666127522785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1,0.08905599514643352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1,0.08900800347328186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1,0.07276266813278198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1,0.07263466715812683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,3,0.08914132912953694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,3,0.08915733297665913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,3,0.07283199826876323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,3,0.07262933254241943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,7,0.09277333815892537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,7,0.09240532914797465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,7,0.0763733337322871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,7,0.07643199960390727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,15,0.09309333562850952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,15,0.09310932954152425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,15,0.07858666777610779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,15,0.07850133379300435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,31,0.1125440001487732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,31,0.11213333408037822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,31,0.09924266735712688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,31,0.09910933176676433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,63,0.11356799801190694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,63,0.11374400059382121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,63,0.099589337905248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,63,0.09937066833178203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,127,0.11359467109044392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,127,0.11354133486747742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,127,0.09948800007502238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,127,0.09911466638247173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,255,0.11543466647466023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,255,0.13397866487503052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,255,0.09961600104967754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,255,0.11984533071517944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,511,0.1155413289864858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,511,0.19957866271336874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,511,0.10087999701499939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,511,0.1865440011024475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1023,0.11436266700426738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1023,0.32870399951934814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1023,0.10008533795674641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1023,0.31650133927663165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,2047,0.11523200074831645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,2047,0.5973920027414957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,2047,0.10097600022951762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,2047,0.5812533299128214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1,0.08921600381533305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1,0.08876799543698628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1,0.07242133220036824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1,0.07282133400440216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,3,0.08912533521652222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,3,0.08904533584912618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,3,0.07235200206438701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,3,0.07271466652552287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,7,0.09563733140627544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,7,0.09660800298055013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,7,0.07628266513347626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,7,0.07658133407433827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,15,0.09405866265296936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,15,0.09433066844940186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,15,0.0788266658782959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,15,0.08141333361466725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,31,0.11561066905657451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,31,0.11356799801190694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,31,0.09929600358009338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,31,0.09909333785374959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,63,0.11347200473149617
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,63,0.11345600088437398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,63,0.09913067022959392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,63,0.09930133819580078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,127,0.11517332990964253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,127,0.11529067158699036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,127,0.09929600358009338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,127,0.09918933113416036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,255,0.11532266934712727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,255,0.1341386636098226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,255,0.09930666287740071
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,255,0.1197760005791982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,511,0.11528533697128296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,511,0.1998080015182495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,511,0.10108266274134318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,511,0.18706132968266806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1023,0.11560533444086711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1023,0.32898133993148804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1023,0.09920533498128255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1023,0.31645333766937256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,2047,0.11567999919255574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,2047,0.6191093524297079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,2047,0.10099200407663982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,2047,0.5819466511408488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1,0.08899733424186707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1,0.08896533648173015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1,0.07445333401362102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1,0.07489066819349925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,3,0.08987200260162354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,3,0.08950933814048767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,3,0.07489066819349925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,3,0.07484266658624013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,7,0.09294933080673218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,7,0.09277333815892537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,7,0.076773335536321
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,7,0.07690666615962982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,15,0.09464533130327861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,15,0.09436266620953877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,15,0.07874133189519246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,15,0.0788213312625885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,31,0.11194666226704915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,31,0.11170666416486104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,31,0.09924800197283427
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,31,0.09907199939092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,63,0.1134986678759257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,63,0.11377066373825073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,63,0.09941333532333374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,63,0.099263995885849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,127,0.11541866262753804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,127,0.11562666296958923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,127,0.10115733742713928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,127,0.10099732875823975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,255,0.11565333604812622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,255,0.1341386636098226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,255,0.09976533055305481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,255,0.1197760005791982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,511,0.11586667100588481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,511,0.19961599508921304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,511,0.10109333197275798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,511,0.18714133898417154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1023,0.11550933122634888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1023,0.32889066139856976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1023,0.10075199604034424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1023,0.3165173331896464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,2047,0.11529599626859029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,2047,0.6058026552200317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,2047,0.10101333260536194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,2047,0.5821919838587443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,3,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,3,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,3,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,3,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,7,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,7,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,7,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,7,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,15,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,15,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,15,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,15,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,31,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,31,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,31,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,31,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,63,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,63,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,63,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,63,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,127,0.01810666670401891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,127,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,127,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,127,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,255,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,255,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,255,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,255,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,511,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,511,0.019765333582957584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,511,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,511,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1023,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1023,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1023,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1023,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,2047,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,2047,0.045610666275024414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,2047,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,2047,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,3,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,3,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,3,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,3,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,7,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,7,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,7,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,7,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,15,0.013776000589132309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,15,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,15,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,15,0.015840000162522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,31,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,31,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,31,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,31,0.017808000246683758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,63,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,63,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,63,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,63,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,127,0.015520000209410986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,127,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,127,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,127,0.021488000949223835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,255,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,255,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,255,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,255,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,511,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,511,0.021664001047611237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,511,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,511,0.023552000522613525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1023,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1023,0.03242133309443792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1023,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1023,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,2047,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,2047,0.053823997577031456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,2047,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,2047,0.054645334680875145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,4095,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,4095,0.09703999757766724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,4095,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,4095,0.09315733114878337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,8191,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,8191,0.18112534284591675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,8191,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,8191,0.17255467176437378
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,16383,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,16383,0.35734931627909344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,16383,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,16383,0.32918934027353924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,3,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,3,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,3,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,3,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,7,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,7,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,7,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,7,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,15,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,15,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,15,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,15,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,31,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,31,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,31,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,31,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,63,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,63,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,63,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,63,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,127,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,127,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,127,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,127,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,255,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,255,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,255,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,255,0.0206986665725708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,511,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,511,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,511,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,511,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1023,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1023,0.033226666351159416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1023,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1023,0.03369066615899404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,2047,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,2047,0.054234668612480164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,2047,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,2047,0.054058666030565895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,4095,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,4095,0.09708799918492635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,4095,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,4095,0.09309333562850952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,8191,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,8191,0.1811786691347758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,8191,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,8191,0.1713013251622518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,16383,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,16383,0.3563786745071411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,16383,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,16383,0.32891732454299927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,3,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,3,0.01458666721979777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,3,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,3,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,7,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,7,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,7,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,7,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,15,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,15,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,15,0.015728000551462173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,15,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,31,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,31,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,31,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,31,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,63,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,63,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,63,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,63,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,127,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,127,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,127,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,127,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,255,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,255,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,255,0.020901332298914593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,255,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,511,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,511,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,511,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,511,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1023,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1023,0.033573334415753685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1023,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1023,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,2047,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,2047,0.054341331124305725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,2047,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,2047,0.05403199791908264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,4095,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,4095,0.09719467163085938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,4095,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,4095,0.09293867150942485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,8191,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,8191,0.18194133043289185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,8191,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,8191,0.17266666889190674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,16383,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,16383,0.35922133922576904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,16383,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,16383,0.3295253316561381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,255,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,511,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,511,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1023,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1023,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,2047,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,2047,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,2047,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,2047,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,4095,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,4095,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,4095,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,4095,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,8191,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,8191,0.021520001192887623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,8191,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,8191,0.01942933350801468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,16383,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,16383,0.027488000690937042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,16383,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,16383,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1,0.17116800944010416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1,0.17094399531682333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1,0.13834666212399802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1,0.13824533422787985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,3,0.17087999979654947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,3,0.17082132895787558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,3,0.13822933038075766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,3,0.138154665629069
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,7,0.17680533727010092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,7,0.17688000202178955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,7,0.1441920002301534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,7,0.14421866337458292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,15,0.17937066157658896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,15,0.17920533816019693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,15,0.15061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,15,0.15051733454068503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,31,0.21785600980122885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,31,0.21767467260360718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,31,0.19047466913859049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,31,0.19030400117238364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,63,0.21988266706466675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,63,0.21943465868631998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,63,0.1914506753285726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,63,0.1914986570676168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,127,0.22142932812372842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,127,0.22108266750971475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,127,0.19331733385721842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,127,0.19352533419926962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,255,0.22210667530695596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,255,0.2605760097503662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,255,0.19326400756835938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,255,0.23269333442052206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,511,0.22221332788467407
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,511,0.39021865526835126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,511,0.19347200791041055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,511,0.3650933504104614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1023,0.22237332661946616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1023,0.6500213146209717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1023,0.1933120091756185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1023,0.6251253286997477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1,0.17113065719604492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1,0.1710240046183268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1,0.13846400380134583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1,0.1381386617819468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,3,0.17091200749079385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,3,0.17090133825937906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,3,0.13823466499646506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,3,0.13820266723632812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,7,0.17682133118311563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,7,0.17677332957585654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,7,0.14411200086275736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,7,0.14405333002408346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,15,0.17917867501576742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,15,0.1789813240369161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,15,0.15028799573580423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,15,0.15051199992497763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,31,0.21779199441274008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,31,0.21781333287556967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,31,0.19126933813095093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,31,0.19105599323908487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,63,0.21996800104777017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,63,0.21978666385014853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,63,0.19133333365122476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,63,0.1913813352584839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,127,0.22169599930445352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,127,0.22192533810933432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,127,0.19316800435384116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,127,0.1934880018234253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,255,0.22216000159581503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,255,0.26096532742182416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,255,0.19347200791041055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,255,0.23405865828196207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,511,0.2221119999885559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,511,0.3902133305867513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,511,0.1933013399442037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,511,0.3651573260625203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1023,0.22241065899531046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1023,0.6485120058059692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1023,0.19372800985972086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1023,0.6793973445892334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1,0.17094399531682333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1,0.1709280014038086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1,0.14018133282661438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1,0.1400053302447001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,3,0.17100799083709717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,3,0.17068266868591309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,3,0.14018666744232178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,3,0.13980266451835632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,7,0.17708800236384073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,7,0.17693867286046347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,7,0.14451199769973755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,7,0.14531733592351279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,15,0.1798400084177653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,15,0.17991467316945395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,15,0.15080533425013223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,15,0.15095999836921692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,31,0.21821333964665732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,31,0.21817066272099814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,31,0.19128533204396567
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,31,0.19127466281255087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,63,0.22000000874201456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,63,0.22004266579945883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,63,0.19152534008026123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,63,0.19147199392318726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,127,0.22178133328755698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,127,0.22181334098180136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,127,0.19345066944758096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,127,0.19335466623306274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,255,0.22206934293111166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,255,0.26090667645136517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,255,0.19352533419926962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,255,0.2341973384221395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,511,0.22245333592096964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,511,0.39047467708587646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,511,0.19349332650502524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,511,0.36551467577616376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1023,0.22238399585088095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1023,0.6625333229700724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1023,0.19369065761566162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1023,0.6248960097630819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1,0.022944000860055287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,3,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,3,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,3,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,3,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,7,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,7,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,7,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,7,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,15,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,15,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,15,0.020975999534130096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,15,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,31,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,31,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,31,0.021509334444999695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,31,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,63,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,63,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,63,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,63,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,127,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,127,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,127,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,127,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,255,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,255,0.02306666721900304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,255,0.02218666672706604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,255,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,511,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,511,0.02916266769170761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,511,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,511,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1023,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1023,0.04200533529122671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1023,0.021546666820844013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1023,0.03738133360942205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,1,0.33451199531555176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,1,0.3345173199971517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,1,0.26759467522303265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,1,0.26798399289449054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,3,0.3346293369928996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,3,0.33444801966349286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,3,0.26764800151189166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,3,0.26790400346120197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,7,0.3447360197703044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,7,0.3447573184967041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,7,0.27958933512369794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,7,0.27948800722757977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,15,0.35123733679453534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,15,0.3511679967244466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,15,0.2919893264770508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,15,0.29312533140182495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,31,0.42683200041453045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,31,0.4270079930623372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,31,0.3733706474304199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,31,0.37329065799713135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,63,0.43116267522176105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,63,0.43089067935943604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,63,0.37601598103841144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,63,0.37598931789398193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,127,0.4346666733423869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,127,0.4352000157038371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,127,0.3794613281885783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,127,0.3792906602223714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,255,0.4373386700948079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,255,0.5136320193608602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,255,0.37973864873250324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,255,0.4601600170135498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,1,0.3343679904937744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,1,0.3346346616744995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,1,0.2693919936815898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,1,0.2699519991874695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,3,0.3346560001373291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,3,0.33453865845998126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,3,0.2694080074628194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,3,0.269375999768575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,7,0.34494932492574054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,7,0.3449759880701701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,7,0.2813439965248108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,7,0.2813919981320699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,15,0.35155200958251953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,15,0.3514560063680013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,15,0.2937013308207194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,15,0.2934453288714091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,31,0.4272213379542033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,31,0.4267946481704712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,31,0.37325334548950195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,31,0.37332268555959064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,63,0.43271998564402264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,63,0.4329119920730591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,63,0.37776533762613934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,63,0.3773706754048665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,127,0.43676265080769855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,127,0.4367946783701579
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,127,0.37914665540059406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,127,0.3795040051142375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,255,0.4373279809951782
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,255,0.5391840140024821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,255,0.37965865929921466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,255,0.45926932493845624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,1,0.33423999945322674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,1,0.3346240123112996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,1,0.2712106704711914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,1,0.27131734291712445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,3,0.33444265524546307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,3,0.33463998635609943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,3,0.27145065863927204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,3,0.27137066920598346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,7,0.34517868359883624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,7,0.3447146813074748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,7,0.28109333912531537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,7,0.2812160054842631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,15,0.35236799716949463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,15,0.35143999258677167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,15,0.29358933369318646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,15,0.29345067342122394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,31,0.42719467480977374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,31,0.4273279905319214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,31,0.3738986651102702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,31,0.3739519913991292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,63,0.4327626625696818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,63,0.43275201320648193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,63,0.37734933694203693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,63,0.3774240016937256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,127,0.43667733669281006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,127,0.43674667676289874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,127,0.37990931669871014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,127,0.3794399897257487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,255,0.4370400110880534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,255,0.5147466659545898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,255,0.37928001085917157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,255,0.4593706528345744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,1,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,1,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,1,0.03182400017976761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,1,0.0354720006386439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,3,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,3,0.033743999898433685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,3,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,3,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,7,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,7,0.03378133227427801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,7,0.0315733328461647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,7,0.03165333221356074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,15,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,15,0.033674667278925575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,15,0.03341866781314214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,15,0.03324266771475474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,31,0.03401066611210505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,31,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,31,0.033530667424201965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,31,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,63,0.03372266640265783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,63,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,63,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,63,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,127,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,127,0.03329066683848699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,127,0.03169599920511246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,127,0.03163733333349228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,255,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,255,0.03521066655715307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,255,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,255,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,1,0.6606880029042562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,1,0.6599466800689697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,1,0.5272479852040609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,1,0.5279306570688883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,3,0.6602773269017538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,3,0.6601813236872355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,3,0.527733325958252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,3,0.5276053349177042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,7,0.681600014368693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,7,0.6808426380157471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,7,0.5535680055618286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,7,0.5534773270289103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,15,0.6952479680379232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,15,0.6953972975413004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,15,0.5782879988352457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,15,0.6054240067799886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,31,0.8492373625437418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,31,0.8462080160776774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,31,0.7381172974904379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,31,0.7592639923095703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,63,0.8554666837056478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,63,0.8565759658813477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,63,0.7458720207214355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,63,0.7459946473439535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,127,0.8637973467508951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,127,0.8636000156402588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,127,0.7507519721984863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,127,0.7505333423614502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,1,0.6583946545918783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,1,0.6583040157953898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,1,0.5310293436050415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,1,0.5309439897537231
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,3,0.6585706472396851
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,3,0.6583199898401896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,3,0.5310346682866415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,3,0.5308800141016642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,7,0.6815893650054932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,7,0.6811253229777018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,7,0.5539893309275309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,7,0.5535093148549398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,15,0.6954613526662191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,15,0.6954346497853597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,15,0.5786186854044596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,15,0.5784480174382528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,31,0.8469546635945638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,31,0.8468266328175863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,31,0.7390933036804199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,31,0.739466667175293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,63,0.8565279642740885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,63,0.8568480014801025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,63,0.7460532983144125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,63,0.7458186944325765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,127,0.8644586404164633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,127,0.8647253513336182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,127,0.7508479754130045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,127,0.7507039705912272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,1,0.6601279973983765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,1,0.6600586573282877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,1,0.534986654917399
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,1,0.5349546670913696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,3,0.6606719891230265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,3,0.6606719891230265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,3,0.5351306597391764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,3,0.5353653430938721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,7,0.682149330774943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,7,0.6820267041524252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,7,0.5543200174967448
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,7,0.5539040168126425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,15,0.6961599985758463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,15,0.6956480344136556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,15,0.5788533290227255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,15,0.578437328338623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,31,0.8473280270894369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,31,0.8471519947052002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,31,0.7398613293965658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,31,0.739834705988566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,63,0.8560853004455566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,63,0.8566133181254069
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,63,0.7459733486175537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,63,0.7459626992543539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,127,0.8666026592254639
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,127,0.8676586945851644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,127,0.7516266504923502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,127,0.7520106633504232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,1,0.058090666929880776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,1,0.05823466678460439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,1,0.05397866666316986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,1,0.05584000051021576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,3,0.05646933118502299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,3,0.056287998954455055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,3,0.05388799806435903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,3,0.0543146679798762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,7,0.05663466453552246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,7,0.056421334544817604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,7,0.05393599967161814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,7,0.05407466491063436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,15,0.0572266678015391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,15,0.05788266658782959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,15,0.05415999889373779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,15,0.054325332244237266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,31,0.05788800120353699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,31,0.05807999769846598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,31,0.054458667834599815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,31,0.05449600021044413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,63,0.05712000032265981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,63,0.05638933181762695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,63,0.05414933462937673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,63,0.054234668612480164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,127,0.058634668588638306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,127,0.058245331048965454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,127,0.05382933219273885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,127,0.05482666691144308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,3,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,3,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,3,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,3,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,7,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,7,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,7,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,7,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,15,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,15,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,15,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,15,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,31,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,31,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,31,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,31,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,63,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,63,0.020773333807786305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,63,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,63,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,127,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,127,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,127,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,127,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,255,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,255,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,255,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,255,0.02094399929046631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,511,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,511,0.03173866619666418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,511,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,511,0.031152000029881794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1023,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1023,0.04979733129342397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1023,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1023,0.047775998711586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,2047,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,2047,0.08691733082135518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,2047,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,2047,0.08455999692281087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,4095,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,4095,0.15847466389338175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,4095,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,4095,0.15648000439008078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,8191,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,8191,0.30189865827560425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,8191,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,8191,0.300053338209788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,16383,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,16383,0.6213813225428263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,16383,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,16383,0.6291520198186239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,3,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,3,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,3,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,3,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,7,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,7,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,7,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,7,0.015477333217859268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,15,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,15,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,15,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,15,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,31,0.020666666328907013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,31,0.02089600016673406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,31,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,31,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,63,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,63,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,63,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,63,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,127,0.02089066555102666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,127,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,127,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,127,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,255,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,255,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,255,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,255,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,511,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,511,0.03146666785081228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,511,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,511,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1023,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1023,0.05040533343950907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1023,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1023,0.04876266419887543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,2047,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,2047,0.08673600355784099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,2047,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,2047,0.08467732866605122
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,4095,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,4095,0.15833600362141928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,4095,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,4095,0.15651733676592508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,8191,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,8191,0.30215466022491455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,8191,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,8191,0.29975465933481854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,16383,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,16383,0.6224426825841268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,16383,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,16383,0.6292266845703125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,3,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,3,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,3,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,3,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,7,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,7,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,7,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,7,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,15,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,15,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,15,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,15,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,31,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,31,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,31,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,31,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,63,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,63,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,63,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,63,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,127,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,127,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,127,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,127,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,255,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,255,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,255,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,255,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,511,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,511,0.03324799984693527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,511,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,511,0.0314026673634847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1023,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1023,0.049925332268079124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1023,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1023,0.04842133323351542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,2047,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,2047,0.08693333466847737
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,2047,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,2047,0.08494399984677632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,4095,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,4095,0.1586026648680369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,4095,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,4095,0.15635200341542563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,8191,0.02072000006834666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,8191,0.30404800176620483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,8191,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,8191,0.29974399010340375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,16383,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,16383,0.6534773508707682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,16383,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,16383,0.6313866774241129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,3,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,3,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,7,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,15,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,63,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,127,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,127,0.012245333443085352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,255,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,255,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,255,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,255,0.012250666817029318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,511,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,511,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,511,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1023,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1023,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,2047,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,2047,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,2047,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,2047,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,4095,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,4095,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,4095,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,8191,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,8191,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,8191,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,8191,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,16383,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,16383,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,16383,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,16383,0.0365280012289683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,1,1.3107999960581462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,1,1.3104426860809326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,1,1.0456213156382244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,1,1.0447733402252197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,3,1.3106506665547688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,3,1.3105440139770508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,3,1.0507946809132893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,3,1.043882687886556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,7,1.3542505900065105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,7,1.3546239535013835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,7,1.0984319845835369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,7,1.134874661763509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,15,1.3839359283447266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,15,1.38373867670695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,15,1.1484906673431396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,15,1.1480906804402669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,31,1.6851840019226074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,31,1.6852213541666667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,31,1.4701439539591472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,31,1.4702879587809246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,63,1.703978697458903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,63,1.7042773564656575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,63,1.4829120635986328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,63,1.4837279319763184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,1,1.3101759751637776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,1,1.30949862798055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,1,1.0528693199157715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,1,1.0542613665262859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,3,1.3102293014526367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,3,1.3099359671274822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,3,1.0556639830271404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,3,1.054912010828654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,7,1.3549386660257976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,7,1.3551252683003743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,7,1.100005308787028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,7,1.1423947016398113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,15,1.3846720059712727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,15,1.3844960530598958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,15,1.1487200260162354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,15,1.1479307015736897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,31,1.6856427192687988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,31,1.6855947176615398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,31,1.4701654116312664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,31,1.4702399571736653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,63,1.7065013249715169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,63,1.70580259958903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,63,1.4840586980183919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,63,1.4842133522033691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,1,1.3131626447041829
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,1,1.313637336095174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,1,1.0627360343933105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,1,1.0625440279642742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,3,1.3141067028045654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,3,1.3141333262125652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,3,1.0617866516113281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,3,1.0638346672058105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,7,1.355306625366211
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,7,1.3556000391642253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,7,1.101482629776001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,7,1.1014453570048015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,15,1.3840640385945637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,15,1.3840586344401042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,15,1.1501599947611492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,15,1.1497653325398762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,31,1.6879253387451172
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,31,1.6880052884419758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,31,1.4716639518737793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,31,1.4703839619954426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,63,1.726736068725586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,63,1.7294559478759766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,63,1.4854453404744465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,63,1.4882453282674153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,1,0.10449600219726562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,1,0.11187733213106792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,1,0.09698667128880818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,1,0.09718400239944458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,3,0.10357333223025005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,3,0.1048959990342458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,3,0.09729066491127014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,3,0.0969546635945638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,7,0.10347732901573181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,7,0.10339200496673584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,7,0.09691199660301208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,7,0.09680533409118652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,15,0.1032960017522176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,15,0.10316800077756245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,15,0.09686400492986043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,15,0.09744532903035481
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,31,0.10424533486366272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,31,0.10436266660690308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,31,0.09680533409118652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,31,0.09690133730570476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,63,0.1029919981956482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,63,0.10339732964833577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,63,0.09711999694506328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,63,0.09719467163085938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,1,2.6414987246195474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,1,2.637653350830078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,1,2.1024959882100425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,1,2.105829397837321
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,3,2.634453296661377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,3,2.6403093338012695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,3,2.1048266092936196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,3,2.10645325978597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,7,2.728666623433431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,7,2.734842618306478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,7,2.203312079111735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,7,2.203546682993571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,15,2.7883733113606772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,15,2.795717239379883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,15,2.303706645965576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,15,2.305397351582845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,31,3.383242607116699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,31,3.3852532704671225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,31,2.9490346908569336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,1,2.6444212595621743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,31,2.9418506622314453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,1,2.1338133811950684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,1,2.638240019480387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,15,0.043807998299598694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,1,2.134559949239095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,3,2.6495253245035806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,3,2.6467787424723306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,3,2.1375412940979004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,3,2.1275946299235025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,7,2.76038392384847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,7,2.760080019632975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,7,2.207594712575277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,7,2.2144266764322915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,15,2.820986747741699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,15,2.824831962585449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,15,2.3215893109639487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,15,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,15,2.3146026929219565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,31,3.4201173782348633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,31,3.4173758824666343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,31,2.9613332748413086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,31,2.9637972513834634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,1,2.1363892555236816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,1,2.152506669362386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,3,2.670351982116699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,3,2.686549186706543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,1,2.6670506795247397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,3,2.1438986460367837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,7,2.806938807169596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,7,2.7805280685424805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,7,2.2314507166544595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,7,2.216426690419515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,3,2.157893339792887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,15,2.9084959030151367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,15,2.9077491760253906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,15,2.3388694127400718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,15,2.341482639312744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,31,3.4723733266194663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,31,3.4782400131225586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,31,2.9801012674967446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,1,0.2011786699295044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,31,2.9944852193196616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,1,0.1959679921468099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,1,0.18398400147755942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,1,0.1848106582959493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,3,0.19537599881490073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,3,0.19530133406321207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,3,0.18398932615915933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,3,0.18418665726979574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,7,0.1951520045598348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,7,0.19580266873041788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,7,0.18453333775202432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,7,0.18471999963124594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,15,0.19550933440526327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,15,0.19608533382415771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,15,0.18516800800959268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,15,0.1848533352216085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,31,0.1957599918047587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,31,0.1853440006573995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1,0.027957332630952198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,31,0.19504533211390176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,31,0.18389334281285605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1,0.0239680012067159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,3,0.027978666126728058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,3,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,3,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,3,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,7,0.029685333371162415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,7,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,7,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,7,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,15,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,15,0.02940800040960312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,15,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,15,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,31,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,31,0.03356799980004629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,31,0.029477333029111225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,31,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,63,0.03373866776625315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,63,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,63,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,63,0.03046933313210805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,127,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,127,0.03410666684309641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,127,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,127,0.031343999008337654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,255,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,255,0.0395413339138031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,255,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,255,0.03631466627120972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,511,0.03376533339420954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,511,0.0563679983218511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,511,0.03164800008138021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,511,0.05395199855168661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1023,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1023,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1023,0.09127466877301534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1023,0.08844266335169475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,2047,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,2047,0.030997333427270252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,2047,0.1627253293991089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,2047,0.15836800138155618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,4095,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,4095,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,4095,0.3020533323287964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,4095,0.298144002755483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,8191,0.033887999753157295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,8191,0.03127466638882955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,8191,0.5827733278274536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1,0.02755733331044515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,8191,0.5783679882685343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1,0.024133334557215374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,3,0.027989332874615986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,3,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,3,0.024357333779335022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,3,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,7,0.02916266769170761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,7,0.02924266705910365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,7,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,7,0.02535466601451238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,15,0.029258665939172108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,15,0.02961066613594691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,15,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,15,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,31,0.033743999898433685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,31,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,31,0.02951466788848241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,31,0.03331733246644338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,63,0.03389866650104523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,63,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,63,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,63,0.031514666974544525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,127,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,127,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,127,0.031082667410373688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,127,0.03133866687615713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,255,0.03378133227427801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,255,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,255,0.03130666663249334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,255,0.03555733213822047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,511,0.03365333378314972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,511,0.05629866818586985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,511,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,511,0.054048001766204834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1023,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1023,0.09106666843096416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1023,0.031514666974544525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1023,0.08880000313123067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,2047,0.03402666747570038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,2047,0.03141333411137263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,2047,0.162773331006368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,4095,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,2047,0.15832533439000449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,4095,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,4095,0.30209600925445557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,4095,0.2983893354733785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,8191,0.03369066615899404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,8191,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,8191,0.582698663075765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1,0.027786667148272198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,8191,0.5788160165150961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1,0.024847999215126038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1,0.024826665719350178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,3,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,3,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,3,0.02478400121132533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,3,0.02372266600529353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,7,0.02921066681543986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,7,0.029279999434947968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,7,0.02502399931351344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,7,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,15,0.029215998947620392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,15,0.029509333272775013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,15,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,31,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,31,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,31,0.0295413335164388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,31,0.029887999097506206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,63,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,63,0.033615998923778534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,63,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,63,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,127,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,127,0.03384533276160558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,127,0.0312266672650973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,255,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,127,0.031498665610949196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,255,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,255,0.03298133363326391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,255,0.037461332976818085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,511,0.033674667278925575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,511,0.03128000100453695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,511,0.05385600030422211
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1023,0.03397866586844126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1023,0.09105066458384196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1023,0.03147733211517334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1023,0.08887466788291931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,2047,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,2047,0.16176000237464905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,2047,0.03142400085926056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,2047,0.15843199690183005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,4095,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,4095,0.3038240075111389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,4095,0.031130666534105938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,4095,0.297818660736084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,8191,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,8191,0.5925386746724447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,8191,0.031258667508761086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,8191,0.5786399841308594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1,0.012229333321253458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,3,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,7,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,15,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,15,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,31,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,31,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,511,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,511,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1023,0.011829332758982977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1023,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1023,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1023,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,2047,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,2047,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,4095,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,4095,0.026928000152111053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,4095,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,4095,0.02347733328739802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,8191,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,8191,0.04181333382924398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,8191,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,8191,0.03551466763019562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1,0.045935998360315956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,3,0.048165331284205117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,3,0.04144000013669332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,7,0.04955733319123586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,7,0.043562665581703186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,15,0.05827199916044871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,15,0.05242133140563965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,31,0.058186665177345276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,31,0.05400000015894572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,63,0.05830933153629303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,63,0.053685332338015236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,127,0.06859200199445088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,127,0.06439466774463654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,255,0.10111467043558757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,255,0.09686932961146037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,511,0.16898133357365927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,511,0.16260266304016113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1023,0.3038133382797241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1023,0.2903680006663005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,2047,0.5695253213246664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,2047,0.5475680033365885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,4095,1.1011146704355876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,4095,1.0590720176696777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1,0.045610666275024414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,3,0.047983999053637184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,3,0.041589332123597465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,7,0.048565333088239036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,7,0.04158399999141693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,15,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,15,0.05398400127887726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,31,0.05793066819508871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,31,0.053786665201187134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,63,0.05813866853713989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,63,0.053183997670809426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,127,0.06857599814732869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,127,0.0644160012404124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,255,0.10176533460617065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,255,0.09681600332260132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,511,0.16875733931859335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,511,0.161381334066391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1023,0.3020159999529521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1023,0.2905973394711812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,2047,0.5694186687469482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,2047,0.5483893156051636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,4095,1.1134080092112224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,4095,1.0594346523284912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1,0.04609066744645437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1,0.039733332892258964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,3,0.047541335225105286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,3,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,7,0.04983466863632202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,7,0.043578664461771645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,15,0.05813866853713989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,15,0.0524479995171229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,31,0.058378666639328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,31,0.05394133428732554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,63,0.05829333265622457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,63,0.05376533170541128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,127,0.06862933437029521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,127,0.06419200201829274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,255,0.1030560036500295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,255,0.09681600332260132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,511,0.16891199350357056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,511,0.16130133469899496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1023,0.3021013339360555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1023,0.28948267300923664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,2047,0.5776000022888184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,2047,0.5470826625823975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,4095,1.1232746442159016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,4095,1.0872053305308025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,3,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,3,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,7,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,7,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,15,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,15,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,31,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,31,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,63,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,63,0.014181333283583323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,127,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,127,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,255,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,511,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,511,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1023,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1023,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,2047,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,2047,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,4095,0.06102400024731954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,4095,0.04357333481311798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,3,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,7,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,15,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,63,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,255,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,255,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,511,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,511,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1023,0.031141333281993866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1023,0.0317546675602595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,2047,0.041834667325019836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,2047,0.041663999358812966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,4095,0.06640533109505971
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,4095,0.06243200103441874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,8191,0.11343999703725179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,8191,0.10140800476074219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,16383,0.20573866367340088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,16383,0.17906665802001953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,3,0.009039999917149544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,7,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,15,0.008821333448092142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,63,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,127,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,127,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,255,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,255,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,511,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,511,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1023,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1023,0.031328000128269196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,2047,0.04186133543650309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,2047,0.042026668787002563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,4095,0.06643733382225037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,4095,0.062037333846092224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,8191,0.11335466305414836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,8191,0.1014453371365865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,16383,0.20668266216913858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,16383,0.17907732725143433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,3,0.008693333094318708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,7,0.008762666955590248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,15,0.008805333326260248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,63,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,127,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,127,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,255,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,255,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,511,0.024677333732446034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,511,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1023,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1023,0.03183999905983607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,2047,0.04208533465862274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,2047,0.04391466577847799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,4095,0.06655466556549072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,4095,0.06246933341026306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,8191,0.11334400375684102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,8191,0.10136000315348308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,16383,0.20579200983047485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,16383,0.17901867628097534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,3,0.011941333611806234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,7,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,7,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,15,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,15,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,31,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,127,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,255,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,255,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,511,0.012373333175977072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1023,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,2047,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,2047,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,4095,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,4095,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,8191,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,8191,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,16383,0.020021333048741024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,16383,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,7,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,15,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,31,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,31,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,63,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,63,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,127,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,255,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,255,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,511,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,511,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1023,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1023,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,2047,0.05428266525268555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,2047,0.0499893327554067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,4095,0.09705600142478943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,4095,0.09102400143941243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,8191,0.1809813380241394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,8191,0.17103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,16383,0.34705066680908203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,16383,0.3312053283055623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,3,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,3,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,7,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,7,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,15,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,31,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,63,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,63,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,127,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,127,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,255,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,255,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,511,0.023034666975339253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,511,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1023,0.03329066683848699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1023,0.02972800036271413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,2047,0.054799998799959816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,2047,0.05003199974695841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,4095,0.09714667002360027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,4095,0.09086400270462036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,8191,0.18116267522176108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,8191,0.17067732413609824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,16383,0.34622931480407715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,16383,0.33084267377853394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,3,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,3,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,7,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,15,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,15,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,31,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,31,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,63,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,63,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,127,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,255,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,255,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,511,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,511,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1023,0.033615998923778534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1023,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,2047,0.05449600021044413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,2047,0.04994666576385498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,4095,0.09718933701515198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,4095,0.09113599856694539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,8191,0.18181333939234415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,8191,0.1711039940516154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,16383,0.34698132673899335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,16383,0.33269333839416504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,7,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,255,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,511,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1023,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,2047,0.016522667060295742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,2047,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,4095,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,4095,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,8191,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,8191,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,16383,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,16383,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1,0.08678399523099263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1,0.07453333338101704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,3,0.08911466598510742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,3,0.07649066547552745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,7,0.09128000338872273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,7,0.08071466783682506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,15,0.11121599872907002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,15,0.0990559955437978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,31,0.11147733529408772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,31,0.09915199875831604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,63,0.11082667112350464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,63,0.09912000099817912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,127,0.13079466422398886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,127,0.11984533071517944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,255,0.19744000832239786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,255,0.1850186586380005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,511,0.32898133993148804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,511,0.3137279947598775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1023,0.5934826532999674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1023,0.5680213371912638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,2047,1.1228853066762288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,2047,1.077344020207723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1,0.08700266480445862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1,0.07462400197982788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,3,0.09081600109736125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,3,0.07668800155321757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,7,0.0909546713034312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,7,0.08041599889596303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,15,0.11142399907112122
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,15,0.09892800450325012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,31,0.11134933431943257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,31,0.0993386705716451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,63,0.11145599683125813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,63,0.09946133693059285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,127,0.13196800152460733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,127,0.11961066722869873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,255,0.1958720088005066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,255,0.18497065703074136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,511,0.32849599917729694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,511,0.31272000074386597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1023,0.5946453412373861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1023,0.5667893489201864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,2047,1.130394697189331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,2047,1.0749866962432861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1,0.08702400326728821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1,0.07442666590213776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,3,0.08904000123341878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,3,0.07633600135644276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,7,0.0912000040213267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,7,0.07898666461308797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,15,0.11125333110491435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,15,0.09915199875831604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,31,0.11137066284815471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,31,0.09922132889429729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,63,0.11161599556605022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,63,0.09935466448465984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,127,0.1317866643269857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,127,0.12079999844233195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,255,0.19761600097020468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,255,0.18513067563374838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,511,0.3275146683057149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,511,0.31353600819905597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1023,0.601312001546224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1023,0.5670453310012817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,2047,1.1440213521321614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,2047,1.0938879648844402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,3,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,3,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,7,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,7,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,15,0.017871999492247898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,15,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,31,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,31,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,63,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,63,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,127,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,127,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,255,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,255,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,511,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,511,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1023,0.03781333317359289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1023,0.02720000098148982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,2047,0.062463998794555664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,2047,0.04549333453178406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,3,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,3,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,7,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,7,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,15,0.014725333700577417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,15,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,31,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,31,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,63,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,63,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,127,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,127,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,255,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,255,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,511,0.0313226655125618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,511,0.03335466732581457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1023,0.05376533170541128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1023,0.05380799869696299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,2047,0.09525866309801738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,2047,0.09297066926956177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,4095,0.1807146668434143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,4095,0.17043199141820273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,8191,0.3500320116678874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,8191,0.3263360063234965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,16383,0.6949813365936279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,16383,0.6356799999872843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,3,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,3,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,7,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,7,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,15,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,15,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,31,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,31,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,63,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,63,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,127,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,127,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,255,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,255,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,511,0.03177600105603536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,511,0.03368533402681351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1023,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1023,0.05433600147565206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,2047,0.09673600395520528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,2047,0.09318932890892029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,4095,0.1832266648610433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,4095,0.17083734273910522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,8191,0.35021865367889404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,8191,0.32635732491811115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,16383,0.7049760023752848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,16383,0.6355786720911661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,3,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,3,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,7,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,7,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,15,0.013653332988421122
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,15,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,31,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,31,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,63,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,63,0.01961600035429001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,127,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,127,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,255,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,255,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,511,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,511,0.03363733241955439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1023,0.054101333022117615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1023,0.055071999629338585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,2047,0.09609066446622212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,2047,0.09356266260147095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,4095,0.18131200472513834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,4095,0.17116800944010416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,8191,0.35181868076324463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,8191,0.326693336168925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,16383,0.7269173463185629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,16383,0.6364959875742594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,255,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,511,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1023,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1023,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,2047,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,2047,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,4095,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,8191,0.022304000953833263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,8191,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,16383,0.037605332831541695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,16383,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1,0.16861865917841592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1,0.14004266262054443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,3,0.17294400930404663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,3,0.14644799629847208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,7,0.17543466885884604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,7,0.15261866648991904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,15,0.21406932671864828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,15,0.19138665994008383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,31,0.21438399950663248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,31,0.19160000483194986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,63,0.2162719964981079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,63,0.19313067197799683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,127,0.25468266010284424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,127,0.2325813372929891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,255,0.38630398114522296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,255,0.36166401704152423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,511,0.6479093233744303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,511,0.6154346863428751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1023,1.1748639742533367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1023,1.1191466649373372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1,0.16704533497492471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1,0.13993066549301147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,3,0.17286400000254312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,3,0.14573333660761514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,7,0.17700799306233725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,7,0.15288000305493674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,15,0.21394666035970053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,15,0.19089599450429282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,31,0.21414933602015176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,31,0.19140799840291342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,63,0.21580799420674643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,63,0.19137599070866904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,127,0.25487999121348065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,127,0.23324799537658691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,255,0.38581331570943195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,255,0.36141331990559894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,511,0.6460586786270142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,511,0.6157653331756592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1023,1.1882133483886719
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1023,1.118389368057251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1,0.1681706706682841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1,0.1402293344338735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,3,0.17319466670354208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,3,0.14619732896486917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,7,0.1767573356628418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,7,0.15240533153216043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,15,0.2139093279838562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,15,0.18992000818252563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,31,0.2137813369433085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,31,0.19141866763432822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,63,0.21590399742126465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,63,0.19131199518839517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,127,0.2545013427734375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,127,0.23267199595769247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,255,0.38489067554473877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,255,0.3612746795018514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,511,0.6603946685791016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,511,0.6159786780675253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1023,1.19705597559611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1023,1.1421279907226562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1,0.02332266668478648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,3,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,3,0.024282666544119518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,7,0.025439999997615814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,7,0.023685333629449207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,15,0.025648000339667004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,15,0.023503998915354412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,31,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,31,0.023434666295846302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,63,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,63,0.023525332411130268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,127,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,127,0.023706667125225067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,255,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,255,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,511,0.03772799919048945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,511,0.02942933390537898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1023,0.0581279993057251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1023,0.04212800165017446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,1,0.3285920023918152
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,1,0.2710826595624288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,3,0.34004799524943036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,3,0.2835413416226705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,7,0.3450453281402588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,7,0.2979733347892761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,15,0.42234134674072266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,15,0.37322131792704266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,31,0.4227840105692546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,31,0.3747946818669637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,63,0.42474667231241864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,63,0.37562668323516846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,127,0.5009013414382935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,127,0.4580800135930379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,255,0.7626026471455892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,255,0.7138400077819824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,1,0.32652799288431805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,1,0.27134933074315387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,3,0.3403306802113851
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,3,0.28377066055933636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,7,0.34683199723561603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,7,0.29758399724960327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,15,0.421994686126709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,15,0.3736586570739746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,31,0.4222559928894043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,31,0.37569065888722736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,63,0.42477333545684814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,63,0.37614933649698895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,127,0.5010026693344116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,127,0.4575200080871582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,255,0.7630613644917806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,255,0.7140426635742188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,1,0.3256266713142395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,1,0.27346134185791016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,3,0.33982932567596436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,3,0.2834133307139079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,7,0.3468159834543864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,7,0.29782400528589886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,15,0.4224426746368408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,15,0.37374401092529297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,31,0.42262399196624756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,31,0.3754560152689616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,63,0.4248533248901367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,63,0.37759466965993244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,127,0.504149317741394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,127,0.45788268248240155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,255,0.7833920319875082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,255,0.7171626885732015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,1,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,1,0.03612799942493439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,3,0.040591999888420105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,3,0.037690666814645134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,7,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,7,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,15,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,15,0.03738133360942205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,31,0.0401706670721372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,31,0.037674665451049805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,63,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,63,0.03738133360942205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,127,0.04167999823888143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,127,0.03734400123357773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,255,0.04381866753101349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,255,0.037445334096749626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,1,0.6481546560923258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,1,0.5357173283894857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,3,0.672111988067627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,3,0.5589760144551595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,7,0.6828479766845703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,7,0.5889120101928711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,15,0.8387146790822347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,15,0.7413653532663981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,31,0.8360693454742432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,31,0.7456106344858805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,63,0.8421226342519125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,63,0.7446880340576172
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,127,0.9943093458811442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,127,0.9080586433410645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,1,0.6462826728820801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,1,0.5354346831639608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,3,0.672655979792277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,3,0.5598773161570231
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,7,0.6869493325551351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,7,0.5900906721750895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,15,0.83732803662618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,15,0.7399199803670248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,31,0.8456532955169678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,31,0.7432959874471029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,63,0.8427840073903402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,63,0.7469759782155355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,127,1.0046133200327556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,127,0.9101706345876058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,1,0.646016001701355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,1,0.5378453334172567
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,3,0.6734186808268229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,3,0.5601226488749186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,7,0.6888319651285807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,7,0.58897598584493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,15,0.8382240136464437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,15,0.740277369817098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,31,0.8400959968566895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,31,0.7427466710408529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,63,0.8507146835327148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,63,0.7474986712137858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,127,1.035312016805013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,127,0.9225066502888998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,1,0.0664160003264745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,1,0.06285333136717479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,3,0.06830400228500366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,3,0.06200533111890157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,7,0.06834133466084798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,7,0.06276266773541768
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,15,0.06837333242098491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,15,0.06273066500822704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,31,0.06836266815662384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,31,0.06297066807746887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,63,0.06810133159160614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,63,0.06218666831652323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,127,0.06861333549022675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,127,0.06437866886456807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,3,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,3,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,7,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,7,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,15,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,15,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,31,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,31,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,63,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,63,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,127,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,127,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,255,0.03153600047032038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,255,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,511,0.04974933465321859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,511,0.04798933366934458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1023,0.08496532837549846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1023,0.08290666838486989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,2047,0.15666666626930237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,2047,0.15409599741299948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,4095,0.30058133602142334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,4095,0.2959146698315938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,8191,0.586517333984375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,8191,0.5779626766840616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,16383,1.2226773103078206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,16383,1.3745813369750977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1,0.017786666750907898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,3,0.01844800015290578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,3,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,7,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,7,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,15,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,15,0.017994667092959087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,31,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,31,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,63,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,63,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,127,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,255,0.031557333966096245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,255,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,511,0.04953599969546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,511,0.047839999198913574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1023,0.0848479966322581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1023,0.08452799916267395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,2047,0.15651733676592508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,2047,0.1543359955151876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,4095,0.3017599980036418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,4095,0.2956586678822835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,8191,0.5908106565475464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,8191,0.5765120188395182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,16383,1.3042559623718262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,16383,1.3743839263916016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1,0.018016000588734944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,3,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,7,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,7,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,15,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,15,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,31,0.019973333925008774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,31,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,63,0.01974933346112569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,63,0.017658667018016178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,127,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,127,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,255,0.03218133250872294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,255,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,511,0.05026133358478546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,511,0.04816000163555145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1023,0.0851146678129832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1023,0.08317333459854126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,2047,0.15687466661135355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,2047,0.1544266641139984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,4095,0.3017706672350566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,4095,0.29550399382909137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,8191,0.5948479970296224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,8191,0.5768959919611613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,16383,1.4126879374186199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,16383,1.5182719230651855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,511,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,511,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1023,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1023,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,2047,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,2047,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,4095,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,4095,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,8191,0.03866666555404663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,8191,0.029114666084448498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,16383,0.06076266864935557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,16383,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,1,1.2969813346862793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,1,1.0711413224538167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,3,1.342458724975586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,3,1.1181600093841553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,7,1.3691466649373372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,7,1.1712213357289631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,15,1.6687893867492676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,15,1.4773866335550945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,31,1.6705439885457356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,31,1.480229377746582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,63,1.6920000712076824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,63,1.4947892824808757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,1,1.2899680137634277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,1,1.0723040103912354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,3,1.3413866360982258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,3,1.1173973083496094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,7,1.374618689219157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,7,1.1868000030517578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,15,1.6753387451171875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,15,1.477952003479004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,31,1.6872480710347493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,31,1.489701271057129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,63,1.7160800298055012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,63,1.508933385213216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,1,1.2877439657847087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,1,1.083674669265747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,3,1.3498934110005696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,3,1.1273492972056072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,7,1.3922026952107747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,7,1.1863306363423665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,15,1.6937653223673503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,15,1.4871519406636555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,31,1.7133866945902507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,31,1.5069546699523926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,63,1.7364427248636882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,63,1.539445400238037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,1,0.12316266695658366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,1,0.11559466520945232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,3,0.12364266316095988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,3,0.11552000045776367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,7,0.123690664768219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,7,0.11574932932853699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,15,0.12338133653004964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,15,0.1149066686630249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,31,0.12378133336702983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,31,0.1150986651579539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,63,0.1232373317082723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,63,0.11458133657773335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,1,2.736138661702474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,1,2.278794606526693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,3,2.8191839853922525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,3,2.3338027000427246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,7,2.8635199864705405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,7,2.4260533650716147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,15,3.4074134826660156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,15,3.0150718688964844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,31,3.40501340230306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,31,3.01798407236735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,1,2.7631308237711587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,1,2.311018625895182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,3,2.8447786966959634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,3,2.352421283721924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,7,2.892149289449056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,7,2.4537386894226074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,15,3.420698801676432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,15,3.033594767252604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,31,3.4206933975219727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,31,3.036656061808268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,1,2.8187840779622397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,1,2.34878937403361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,3,2.884101231892904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,3,2.389413356781006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,7,2.9293654759724936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,7,2.484389305114746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,15,3.4371093114217124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,15,3.0438667933146157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,31,3.4382346471150718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,31,3.053466796875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,1,0.23321600755055746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,1,0.21683200200398764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,3,0.23354132970174155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,3,0.21685334046681723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,7,0.23226133982340494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,7,0.21600000063578287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,15,0.2344693342844645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,15,0.2168160080909729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,31,0.23427200317382812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,31,0.21636799971262613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1,0.02346133440732956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,3,0.027647999425729115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,3,0.02497066557407379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,7,0.028021333118279774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,7,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,15,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,15,0.029616000751654308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,31,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,31,0.029674666623274486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,63,0.03344533344109853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,63,0.030773334205150604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,127,0.03797333439191183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,127,0.03580799947182337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,255,0.05614933371543884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,255,0.054117331902186074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,511,0.09106133381525676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,511,0.08714666962623596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1023,0.1604586640993754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1023,0.15612266461054483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,2047,0.2997973362604777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,2047,0.2903199990590413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,4095,0.5783306757609049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,4095,0.5599413315455118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,8191,1.1363413333892822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,8191,1.1055786609649658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,3,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,3,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,7,0.027765333652496338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,7,0.025029333929220837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,15,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,15,0.029535998900731403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,31,0.033258666594823204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,31,0.02958400050799052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,63,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,63,0.02959466725587845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,127,0.0377866675456365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,127,0.03535466641187668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,255,0.05600533386071523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,255,0.05234666665395101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,511,0.09081600109736125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,511,0.08682133754094441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1023,0.16087999939918518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1023,0.154639999071757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,2047,0.30004799365997314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,2047,0.29029866059621173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,4095,0.5795093377431234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,4095,0.5597759882609049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,8191,1.1547306378682454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,8191,1.1055839856465657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1,0.02385066697994868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,3,0.027285332481066387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,3,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,7,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,7,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,15,0.033376000821590424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,15,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,31,0.03317866722742716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,31,0.02938133229811986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,63,0.03316266586383184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,63,0.031770666440327965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,127,0.03908266623814901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,127,0.03613866617282232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,255,0.05683733522891998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,255,0.05398933092753092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,511,0.09116266171137492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,511,0.08740267157554626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1023,0.16065067052841187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1023,0.1550826629002889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,2047,0.3001120090484619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,2047,0.29208000500996906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,4095,0.5908159812291464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,4095,0.5622026522954305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,8191,1.167423963546753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,8191,1.1239840189615886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,3,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,7,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,7,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,15,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,31,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,63,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,127,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,255,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,255,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,511,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,511,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1023,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1023,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,2047,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,2047,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,4095,0.037871999045213066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,4095,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,8191,0.05970133344332377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,8191,0.04168533285458883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1,0.03746666759252548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1,0.03763733307520548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1,0.03363733241955439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,3,0.037605332831541695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,3,0.03748800108830134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,3,0.03330666571855545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,3,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,7,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,7,0.039701332648595176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,7,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,7,0.033402666449546814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,15,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,15,0.0394400010506312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,15,0.03570133447647095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,15,0.034330666065216064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,31,0.04769066472848257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,31,0.047040000557899475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,31,0.04181866844495138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,31,0.041434665520985924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,63,0.04780800143877665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,63,0.049029335379600525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,63,0.0418453315893809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,63,0.04165333261092504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,127,0.0476693312327067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,127,0.04808000226815542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,127,0.04204800228277842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,127,0.04364266494909922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,255,0.0480320006608963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,255,0.056159997979799904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,255,0.04381333291530609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,255,0.05035200218359629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,511,0.047872001926104225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,511,0.08081600069999695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,511,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,511,0.0764213353395462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1023,0.047744000951449074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1023,0.13209600249926248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1023,0.041706666350364685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1023,0.12774399916330972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,2047,0.047637333472569786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,2047,0.23662400245666504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,2047,0.041877334316571556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,2047,0.23227200905481973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,4095,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,4095,0.44654401143391925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,4095,0.04171200096607208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,4095,0.43933331966400146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1,0.03771200031042099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1,0.037946666280428566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,1,0.033189333975315094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,3,0.03751999884843826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,3,0.0376800000667572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,3,0.0335359995563825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,3,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,7,0.03926933308442434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,7,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,7,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,7,0.03428266694148382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,15,0.04043200115362803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,15,0.04016000032424927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,15,0.03513599932193756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,15,0.03588266670703888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,31,0.04799999793370565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,31,0.04751466711362203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,31,0.04205333193143209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,31,0.042064001162846885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,63,0.048101335763931274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,63,0.04782933493455251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,63,0.04221333563327789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,63,0.041797334949175514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,127,0.048154667019844055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,127,0.04809600114822388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,127,0.04187199970086416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,127,0.0421013335386912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,255,0.047839999198913574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,255,0.05583466589450836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,255,0.04206933577855428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,255,0.0498933345079422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,511,0.04782933493455251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,511,0.08072533210118611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,511,0.04348800083001455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,511,0.07655466596285503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1023,0.04804266492525736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1023,0.13369066516558328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,1023,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1023,0.1279306709766388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,2047,0.04795200129350027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,2047,0.23858133951822916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,2047,0.04160533348719279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,2047,0.2323840061823527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,4095,0.04772266745567322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,4095,0.4545653263727824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,4095,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,4095,0.43885334332784015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,3,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,3,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,3,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,3,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,7,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,7,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,7,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,7,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,15,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,15,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,15,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,15,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,31,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,31,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,31,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,31,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,63,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,63,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,63,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,127,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,127,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,127,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,255,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,255,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,255,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,255,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,511,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,511,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,511,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1023,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1023,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1023,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,2047,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,2047,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,2047,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,2047,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,4095,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,4095,0.04384533564249674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,4095,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,4095,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1,0.008938666433095932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,3,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,3,0.00916800027092298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,3,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,7,0.00933333362142245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,7,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,15,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,15,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,15,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,31,0.009946666657924652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,31,0.009808000177145004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,63,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,63,0.011952000359694162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,63,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,127,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,127,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,255,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,255,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,511,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,511,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,511,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,511,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1023,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1023,0.023498666783173878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1023,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1023,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,2047,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,2047,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,2047,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,2047,0.03166399896144867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,4095,0.019600000232458115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,4095,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,4095,0.02347733328739802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,4095,0.04170133173465729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,8191,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,8191,0.0642986645301183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,8191,0.023711999257405598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,8191,0.06187200049559275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,16383,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,16383,0.11152000228563945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,16383,0.024698667228221893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,16383,0.10146666566530864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,3,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,3,0.008816000074148178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,7,0.009205333267649015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,7,0.009152000149091085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,7,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,15,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,15,0.008714666590094566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,31,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,31,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,127,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,255,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,255,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,511,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,511,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,511,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,511,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1023,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1023,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,1023,0.0235359991590182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1023,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,2047,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,2047,0.02922133356332779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,2047,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,2047,0.0314026673634847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,4095,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,4095,0.042090664307276406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,4095,0.02346666653951009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,4095,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,8191,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,8191,0.06607466439406078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,8191,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,8191,0.06241600215435028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,16383,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,16383,0.11166399717330933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,16383,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,16383,0.10123200217882793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,3,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,3,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,7,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,7,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,15,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,15,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,15,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,31,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,31,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,63,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,511,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,511,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1023,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1023,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,2047,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,2047,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,2047,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,2047,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,4095,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,4095,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,4095,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,4095,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,8191,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,8191,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,8191,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,8191,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,16383,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,16383,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,16383,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,16383,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1,0.01381333296497663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1,0.015882667154073715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,3,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,3,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,3,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,3,0.01628799984852473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,7,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,7,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,7,0.01575999955336253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,7,0.016229332735141117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,15,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,15,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,15,0.01581866666674614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,15,0.016197333733240765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,31,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,31,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,31,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,31,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,63,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,63,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,63,0.018229333062966663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,63,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,127,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,127,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,127,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,127,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,255,0.016458666572968166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,255,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,255,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,255,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,511,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,511,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,511,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,511,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1023,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1023,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1023,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1023,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,2047,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,2047,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,2047,0.023024000227451324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,2047,0.03536533315976461
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,4095,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,4095,0.055045331517855324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,4095,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,4095,0.05507733424504598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,8191,0.01573866605758667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,8191,0.09758933385213216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,8191,0.021615999440352123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,8191,0.09345066547393799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,16383,0.016202667107184727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,16383,0.1810986598332723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,16383,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,16383,0.16867733001708984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1,0.013882666826248169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1,0.013562666873137156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,1,0.016000000139077503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,3,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,3,0.01589866727590561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,3,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,7,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,7,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,7,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,7,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,15,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,15,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,15,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,15,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,31,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,31,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,31,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,31,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,63,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,63,0.01793066660563151
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,63,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,127,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,127,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,127,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,127,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,255,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,255,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,255,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,255,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,511,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,511,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,511,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,511,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1023,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1023,0.022986667851607006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,1023,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1023,0.02535466601451238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,2047,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,2047,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,2047,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,2047,0.03547733277082443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,4095,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,4095,0.055813332398732506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,4095,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,4095,0.05409599840641022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,8191,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,8191,0.09714667002360027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,8191,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,8191,0.09297066926956177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,16383,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,16383,0.18095467487970987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,16383,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,16383,0.1687893271446228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,3,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,3,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,7,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,63,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,127,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,255,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,255,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,255,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,255,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,511,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,511,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1023,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1023,0.011610666910807291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1023,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1023,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,2047,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,2047,0.012373333175977072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,2047,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,4095,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,4095,0.018570666511853535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,4095,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,4095,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,8191,0.012383999923865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,8191,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,8191,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,8191,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,16383,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,16383,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,16383,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,16383,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1,0.06871466835339864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1,0.06849066913127899
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1,0.05634133517742157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1,0.05635733405749003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,3,0.06842666864395142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,3,0.06844266752401988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,3,0.05653333167235056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,3,0.056320001681645714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,7,0.07052266597747803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,7,0.07060799996058147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,7,0.05841066439946493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,7,0.058277333776156105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,15,0.07230400045712788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,15,0.07252799967924754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,15,0.060346667965253196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,15,0.06021333237489065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,31,0.08688533306121826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,31,0.0849226713180542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,31,0.07666133344173431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,31,0.07673066854476929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,63,0.08683733145395915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,63,0.0867786705493927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,63,0.0766293356815974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,63,0.07664533456166585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,127,0.08687466382980347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,127,0.08715732892354329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,127,0.07687999804814656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,127,0.07656533519426982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,255,0.08660266796747844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,255,0.1030560036500295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,255,0.07698666552702586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,255,0.09299199779828389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,511,0.08845866719881694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,511,0.15244799852371216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,511,0.07675733168919881
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,511,0.14220266540845236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1023,0.08835732936859131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1023,0.25086400906244916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1023,0.0765119989713033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1023,0.2425866723060608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,2047,0.08855467041333516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,2047,0.45398398240407306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,2047,0.07841599980990092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,2047,0.4432106812795003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1,0.06865600248177846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1,0.06860800087451935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,1,0.05801600217819214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1,0.05764266848564148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,3,0.06862399975458781
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,3,0.06849066913127899
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,3,0.05657599866390228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,3,0.05629333357016245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,7,0.07055466870466869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,7,0.07057066758473714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,7,0.05816000203291575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,7,0.05819733440876007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,15,0.07227733234564464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,15,0.07242133220036824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,15,0.06053866446018219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,15,0.06002666552861532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,31,0.08506666620572408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,31,0.0857919951279958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,31,0.0763626645008723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,31,0.0765173335870107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,63,0.08658666412035625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,63,0.08688533306121826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,63,0.07657066484292348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,63,0.07643199960390727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,127,0.08676266670227051
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,127,0.08853866656621297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,127,0.07760000228881836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,127,0.07650133470694225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,255,0.08717333277066548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,255,0.10314666231473286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,255,0.0765226682027181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,255,0.09304533402125041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,511,0.08880000313123067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,511,0.1524799962838491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,511,0.07810133198897044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,1,2.673669179280599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,511,0.14246933658917746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1023,0.08714133501052856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1023,0.2521493236223857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,1023,0.07679999868075053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,2047,0.08715200424194336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1023,0.24254933993021646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,2047,0.07843199868996938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,2047,0.46165867646535236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,2047,0.4429333209991455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,3,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,3,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,3,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,3,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,7,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,7,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,7,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,7,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,15,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,15,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,15,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,15,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,31,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,31,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,31,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,31,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,63,0.015605332950750986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,63,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,63,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,63,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,127,0.016629333297411602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,127,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,127,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,127,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,255,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,255,0.016063999384641647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,255,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,255,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,511,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,511,0.01953599974513054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,511,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,511,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1023,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1023,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1023,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1023,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,2047,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,2047,0.0446720023949941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,2047,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,2047,0.037834666669368744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1,0.019461333751678467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,3,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,3,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,3,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,3,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,7,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,7,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,7,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,7,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,15,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,15,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,15,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,15,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,31,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,31,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,31,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,31,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,63,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,63,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,63,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,63,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,127,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,127,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,127,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,127,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,255,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,255,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,255,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,255,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,511,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,511,0.02499733368555705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,511,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,511,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1023,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1023,0.03346666693687439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1023,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1023,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,2047,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,2047,0.0497920016447703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,2047,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,2047,0.042165334026018776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,4095,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,4095,0.08060800035794576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,4095,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,4095,0.07308266560236613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,8191,0.02187199890613556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,8191,0.14451199769973755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,8191,0.0179626668492953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,8191,0.13222400347391763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,16383,0.021530665457248688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,16383,0.2674613396326701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,16383,0.020101333657900494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,16383,0.250709335009257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,1,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,3,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,3,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,3,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,3,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,7,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,7,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,7,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,7,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,15,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,15,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,15,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,15,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,31,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,31,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,31,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,31,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,63,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,63,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,63,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,63,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,127,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,127,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,127,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,127,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,255,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,255,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,255,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,255,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,511,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,511,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,511,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,511,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1023,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1023,0.0320266659061114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,1023,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1023,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,2047,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,2047,0.05020266771316528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,2047,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,2047,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,4095,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,4095,0.08067200084527333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,4095,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,4095,0.07258133093516032
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,8191,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,8191,0.14306132992108664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,8191,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,8191,0.1320746640364329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,16383,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,16383,0.2680373390515645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,16383,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,16383,0.2508959968884786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,7,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,63,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,127,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,255,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,255,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,15,0.025839999318122864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,511,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,511,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,511,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1023,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1023,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1023,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,2047,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,2047,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,2047,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,2047,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,4095,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,4095,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,4095,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,4095,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,8191,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,8191,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,8191,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,8191,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,16383,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,16383,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,16383,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,16383,0.024832000335057575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1,0.12982933719952902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1,0.1300159990787506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1,0.10546132922172546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1,0.10667733351389568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,3,0.13005333145459494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,3,0.1300106644630432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,3,0.10583466291427612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,3,0.1055519978205363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,7,0.13395733634630838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,7,0.13429866234461466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,7,0.10967466235160828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,7,0.11132267117500305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,15,0.13717866937319437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,15,0.1367093324661255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,15,0.11531200011571248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,15,0.1153546671072642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,31,0.1645813286304474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,31,0.16472533345222473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,31,0.1443893313407898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,31,0.14542399843533835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,63,0.16690667470296225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,63,0.1669279932975769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,63,0.14627200365066528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,63,0.14646933476130167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,127,0.16754666964213052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,127,0.1673813263575236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,127,0.14640532930692038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,127,0.14664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,255,0.16874667008717856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,255,0.19758933782577515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,255,0.1463573376337687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,255,0.17733333508173624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,511,0.1688800056775411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,511,0.2958400050799052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,511,0.1463093360265096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,511,0.27748266855875653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1023,0.16900267203648886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1023,0.14662399888038635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1023,0.490389347076416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1,0.12995733817418417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1023,0.4713919957478841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1,0.13010133306185404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,1,0.10724266370137532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1,0.10738133390744527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,3,0.1297866702079773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,3,0.13011200229326883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,3,0.10712533195813496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,3,0.10726933677991231
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,7,0.13408000270525613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,7,0.13432000080744425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,7,0.10970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,7,0.11131200194358826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,15,0.13618133465449014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,15,0.1378613313039144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,15,0.11563199758529663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,15,0.11560533444086711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,31,0.1644533375898997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,31,0.16482133666674295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,31,0.14483732978502908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,31,0.14607999722162882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,63,0.16673066218694052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,63,0.16674133141835532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,63,0.14633599917093912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,63,0.14620799819628397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,127,0.16863999764124551
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,127,0.1688106656074524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,127,0.1476426621278127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,127,0.14782933394114176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,255,0.16885334253311157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,255,0.19738666216532388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,255,0.14818666378657022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,255,0.1771199901898702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,511,0.1688693364461263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,511,0.2956373294194539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,511,0.14839999874432883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,511,0.27721599737803143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1023,0.16873600085576376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,1023,0.14631999532381693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1023,0.5007040103276571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1023,0.4702506860097249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,3,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,3,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,3,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,3,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,7,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,7,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,7,0.02149333308140437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,7,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,15,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,15,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,15,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,31,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,15,0.021557333568731945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,31,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,31,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,31,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,63,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,63,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,63,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,63,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,127,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,127,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,127,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,127,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,255,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,255,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,255,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,255,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,511,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,511,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,511,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,511,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1023,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1023,0.04178133110205332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1023,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1023,0.035642666121323906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,1,0.2531573375066121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,1,0.2530933419863383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,1,0.20375466346740723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,1,0.20382932821909586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,3,0.2532426714897156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,3,0.252618670463562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,3,0.203658660252889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,3,0.20406399170557657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,7,0.26078933477401733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,7,0.2610986630121867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,7,0.21237866083780924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,7,0.21217066049575806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,15,0.26716800530751544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,15,0.26705066363016766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,15,0.22202134132385254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,15,0.22217067082722983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,31,0.32259200016657513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,31,0.3234506646792094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,31,0.2818186680475871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,31,0.28193066517512005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,63,0.32651732365290326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,63,0.32656000057856244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,63,0.2856106758117676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,63,0.28495466709136963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,127,0.32891732454299927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,127,0.3285546700159709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,127,0.2877279917399089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,127,0.2855520049730937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,255,0.33025066057840985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,255,0.3860693375269572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,255,0.2855679988861084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,255,0.34699734052022296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,1,0.2532693346341451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,1,0.25269333521525067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,1,0.20569600661595663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,1,0.20575465758641562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,3,0.2529760003089905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,3,0.25299733877182007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,3,0.20573333899180093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,3,0.2055893341700236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,7,0.2611946662267049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,7,0.26105600595474243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,7,0.21378666162490845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,7,0.2136746644973755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,15,0.2670346697171529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,15,0.26712532838185626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,15,0.22221867243448892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,15,0.22200000286102295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,31,0.3226720094680786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,31,0.32259200016657513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,31,0.2831573287645976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,31,0.2830186684926351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,63,0.3264960050582886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,63,0.32652799288431805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,63,0.28569066524505615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,63,0.28551467259724933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,127,0.3285973270734151
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,127,0.32975467046101886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,127,0.2874240080515544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,127,0.28729067246119183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,255,0.32877333958943683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,255,0.38820799191792804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,255,0.28587732712427777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,255,0.34675200780232746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,1,0.03279466678698858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,1,0.032960000137488045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,1,0.031290667752424874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,1,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,3,0.0332640012105306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,3,0.033701332906881966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,3,0.03130666663249334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,3,0.03274133304754893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,7,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,7,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,7,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,7,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,15,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,15,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,15,0.0316746657093366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,15,0.031258667508761086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,31,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,31,0.03160000095764796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,31,0.03142400085926056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,31,0.03143466760714849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,63,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,63,0.031658666829268135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,63,0.031301334500312805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,63,0.03145066648721695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,127,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,127,0.033573334415753685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,127,0.03137599925200144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,127,0.03181333343187968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,255,0.033376000821590424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,255,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,255,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,255,0.03162133445342382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,1,0.4987359841664632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,1,0.4976693391799927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,1,0.40008000532786053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,1,0.3985226551691691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,3,0.4983946482340495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,3,0.4986559947331746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,3,0.3982613484064738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,3,0.3982400099436442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,7,0.5143839915593466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,7,0.5151946544647217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,7,0.41706665356953937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,7,0.41765332221984863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,15,0.5247146685918173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,15,0.5256106853485107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,15,0.4355039993921916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,15,0.43724266688028973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,31,0.6377439896265665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,31,0.6385813156763712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,31,0.5577600002288818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,31,0.556874672571818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,63,0.6444799900054932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,63,0.6440320014953613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,63,0.5617973407109579
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,63,0.5625813404719034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,127,0.6508800188700358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,127,0.6500373284022013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,127,0.5646453301111857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,127,0.5647786855697632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,1,0.49771201610565186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,1,0.4985440174738566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,1,0.40437865257263184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,1,0.40375999609629315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,3,0.4996960163116455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,3,0.4987786610921224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,3,0.4045493205388387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,3,0.4045706590016683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,7,0.5152959823608398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,7,0.5152639945348104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,7,0.419050653775533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,7,0.41911999384562176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,15,0.525002678235372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,15,0.5257759888966879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,15,0.4375466505686442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,15,0.4367786645889282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,31,0.6376266479492188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,31,0.639568010965983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,31,0.5572906732559204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,31,0.5582133531570435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,63,0.6459733247756958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,63,0.6458826859792074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,63,0.5620746612548828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,63,0.5622026522954305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,127,0.6562026739120483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,127,0.6583146651585897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,127,0.5660426616668701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,127,0.5646986564000448
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,1,0.05442666510740916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,1,0.05379733443260193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,1,0.052069331208864846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,1,0.052186667919158936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,3,0.05431999762852987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,3,0.054144000013669334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,3,0.05180266499519348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,3,0.05166399975617727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,7,0.05426666637261709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,7,0.05413866539796194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,7,0.05206400156021118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,7,0.05179200073083242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,15,0.054229333996772766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,15,0.0543093333641688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,15,0.052058666944503784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,15,0.052298665046691895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,31,0.05618133147557577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,31,0.05505066613356272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,31,0.05256533126036326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,31,0.05198933184146881
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,63,0.054431999723116554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,63,0.05446400245030721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,63,0.05213333169619242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,63,0.05221866567929586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,127,0.05602133274078369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,127,0.05605866511662801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,127,0.05194666484991709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,127,0.052042668064435325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,3,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,3,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,3,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,7,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,7,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,7,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,15,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,15,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,15,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,15,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,31,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,31,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,31,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,31,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,63,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,63,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,63,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,63,0.016704000532627106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,127,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,127,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,127,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,127,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,255,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,255,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,255,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,255,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,511,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,511,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,511,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,511,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1023,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1023,0.04379733403523763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1023,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1023,0.04186133543650309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,2047,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,2047,0.07472533484299977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,2047,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,2047,0.07248533268769582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,4095,0.13641066352526346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,4095,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,4095,0.13405866424242655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,8191,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,8191,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,8191,0.26051199436187744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,8191,0.2585013310114543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,16383,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,16383,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,16383,0.5247040192286173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,16383,0.5344640016555786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,1,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,3,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,3,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,3,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,3,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,7,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,7,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,7,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,7,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,15,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,15,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,15,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,15,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,31,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,31,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,31,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,31,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,63,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,63,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,63,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,63,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,127,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,127,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,127,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,127,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,255,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,255,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,255,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,255,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,511,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,511,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,511,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,511,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1023,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1023,0.04345066845417023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,1023,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1023,0.04194133480389913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,2047,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,2047,0.07433066765467326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,2047,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,2047,0.0724426656961441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,4095,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,4095,0.13775466879208884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,4095,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,4095,0.13396267096201578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,8191,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,8191,0.26266666253407794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,8191,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,8191,0.25989333788553876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,16383,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,16383,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,16383,0.5377013285954794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,16383,0.5354986588160197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,3,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,3,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,7,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,7,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,7,0.011893333246310553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,15,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,15,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,31,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,31,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,127,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,255,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,511,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,511,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1023,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1023,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,2047,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,2047,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,2047,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,2047,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,4095,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,4095,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,4095,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,4095,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,8191,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,8191,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,8191,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,8191,0.025786665578683216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,16383,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,16383,0.04387733340263367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,16383,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,16383,0.0366239994764328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,1,0.9892213344573975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,1,0.9887519677480062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,1,0.7897546291351318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,1,0.7893333435058594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,3,0.9871306419372559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,3,0.9871413707733154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,3,0.7900746663411459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,3,0.7892746925354004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,7,1.0217546621958415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,7,1.021338701248169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,7,0.8289653460184733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,7,0.8269333044687907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,15,1.0427199999491374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,15,1.0432373682657878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,15,0.865605354309082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,15,0.8648746808369955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,31,1.2672213713328044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,31,1.2690666516621907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,31,1.1152746677398682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,31,1.1050773461659749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,63,1.2817333539326985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,63,1.2823306719462078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,63,1.117850701014201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,63,1.1155306498209636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,1,1.0019466876983643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,1,0.9882559776306152
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,1,0.8015680313110352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,1,0.7999359766642252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,3,0.9890240033467611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,3,0.9887519677480062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,3,0.8032000064849854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,3,0.7990612983703613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,7,1.0223466555277507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,7,1.0226293404897053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,7,0.8313706715901693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,7,0.8294506867726644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,15,1.0433812936147053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,15,1.043573300043742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,15,0.865781307220459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,15,0.8659679889678955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,31,1.2704532941182454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,31,1.2701706886291504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,31,1.1072159608205159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,31,1.1066666444142659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,63,1.2989919980367024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,63,1.3022346496582031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,63,1.1184639930725098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,63,1.1160906950632732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,1,0.09913067022959392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,1,0.09447999795277913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,1,0.09292800227801006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,3,0.09960533181826274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,3,0.09920533498128255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,3,0.09321066737174988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,3,0.0930613378683726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,7,0.1011199951171875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,7,0.09988266229629517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,7,0.0932426651318868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,7,0.09308800101280212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,15,0.10123200217882793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,15,0.10070932904879253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,15,0.09296533465385437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,15,0.0929813285668691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,31,0.09944533308347066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,31,0.10059733192125957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,31,0.09294399619102478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,31,0.09296533465385437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,63,0.10126933455467224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,63,0.09940266609191895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,63,0.09300800164540608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,63,0.0932426651318868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,1,1.9768479665120442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,1,1.9769652684529622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,1,1.5799306233723958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,1,1.579114596048991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,3,1.9830506642659504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,3,1.9805973370869954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,3,1.5817813873291016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,3,1.5809812545776367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,7,2.0500213305155435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,7,2.0515626271565757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,7,1.6527199745178223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,7,1.6525386174519856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,15,2.0941227277119956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,15,2.0970346132914224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,15,1.730170726776123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,15,1.731386661529541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,31,2.539792060852051
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,31,2.543392022450765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,31,2.211712042490641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,31,2.2106773058573403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,1,1.9906400044759114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,1,1.9865652720133464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,1,1.6044853528340657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,1,1.6054666837056477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,3,2.0028692881266275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,3,1.9983253479003906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,3,1.607850710550944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,3,1.6039466857910156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,7,2.0835787455240884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,7,2.0737226804097495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,7,1.6615254084269206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,7,1.6630880037943523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,15,2.16481606165568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,15,2.1438560485839844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,15,1.742575963338216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,15,1.7486133575439453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,31,2.6117067337036133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,31,2.6020053227742515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,31,2.230021317799886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,31,2.23689603805542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,1,0.19153066476186117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,1,0.18940265973409018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,1,0.17774933576583862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,1,0.17738133668899536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,3,0.190394659837087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,3,0.1902559995651245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,3,0.1768746574719747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,3,0.17729065815607706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,7,0.18951465686162314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,7,0.18965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,7,0.17696533600489298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,7,0.17701866229375204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,15,0.18931732575098673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,15,0.18939199050267538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,15,0.17694934209187826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,15,0.17705066998799643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,31,0.18917866547902426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,31,0.18971733252207437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,31,0.1771519978841146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,31,0.1768640081087748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,3,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,3,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,3,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,3,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,7,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,7,0.02306666721900304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,7,0.019637333850065868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,7,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,15,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,15,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,15,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,15,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,31,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,31,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,31,0.02510933329661687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,63,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,63,0.027130665878454845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,63,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,63,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,127,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,127,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,127,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,127,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,255,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,255,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,255,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,255,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,511,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,511,0.0458133320013682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,511,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,511,0.043322667479515076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1023,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1023,0.07246933380762736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1023,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1023,0.07037333150704701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,2047,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,2047,0.12778133153915405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,2047,0.025557334224383037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,2047,0.12584533294041952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,4095,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,4095,0.24005866050720215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,4095,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,4095,0.2366186579068502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,8191,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,8191,0.46166400114695233
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,8191,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,8191,0.45710400740305585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1,0.023354666928450268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,1,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,3,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,3,0.022970666488011677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,3,0.01945066700379054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,3,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,7,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,7,0.02385599911212921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,7,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,7,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,15,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,15,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,15,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,15,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,31,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,31,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,31,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,31,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,63,0.02762666592995326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,63,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,63,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,63,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,127,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,127,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,127,0.02493866781393687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,127,0.025722667574882507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,255,0.028010666370391846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,255,0.03197333216667175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,255,0.025962665677070618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,255,0.030202666918436687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,511,0.029493334392706554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,511,0.04637866715590159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,511,0.02587733417749405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,511,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1023,0.02794133375088374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1023,0.07336533566315968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,1023,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1023,0.07077866792678833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,2047,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,2047,0.1286079982916514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,2047,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,2047,0.12602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,4095,0.02762666592995326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,4095,0.24051199356714884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,4095,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,4095,0.2365973393122355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,8191,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,8191,0.4679946502049764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,8191,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,8191,0.4578239917755127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,7,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,15,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,15,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,63,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,63,0.011621333658695221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,511,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,511,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1023,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1023,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,2047,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,2047,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,2047,0.014544000228246054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,2047,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,4095,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,4095,0.0271519993742307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,4095,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,4095,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,8191,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,8191,0.04326933125654856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,8191,0.014384000251690546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,8191,0.037733333806196846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1,0.03746666759252548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,3,0.03742400060097376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,3,0.03363733241955439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,7,0.03765333443880081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,7,0.035360001027584076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,15,0.045909335215886436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,15,0.041738669077555336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,31,0.04585599899291992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,31,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,63,0.045754666129748024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,63,0.04181333382924398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,127,0.05421866476535797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,127,0.050053333242734276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,255,0.08053866525491078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,255,0.07656000057856242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,511,0.13211733102798462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,511,0.12594133615493774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1023,0.23568532864252725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1023,0.22614399592081705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,2047,0.44305598735809326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,2047,0.4270613193511963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,4095,0.8592480023701986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,4095,0.8284906546274821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1,0.03772799919048945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1,0.03412266572316488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,3,0.03801066676775614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,3,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,7,0.03826133410135905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,7,0.03558400024970373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,15,0.04606399933497111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,15,0.04204266766707102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,31,0.04621866842110952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,31,0.04207466542720795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,63,0.04640000065167745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,63,0.0417546679576238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,127,0.054117331902186074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,127,0.04996266464392344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,255,0.08097599943478902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,255,0.07670400043328603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,511,0.13195733229319254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,511,0.12610666950543722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1023,0.2365600069363912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1023,0.22592000166575113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,2047,0.4501386483510335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,2047,0.42742399374643963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,4095,0.88045867284139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,4095,0.8427680333455404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1,0.013584000368913015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,3,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,3,0.013786666095256805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,7,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,7,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,15,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,15,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,31,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,31,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,63,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,63,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,127,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,127,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,255,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,255,0.013909333695967993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,511,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,511,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1023,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1023,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,2047,0.038319999972979225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,2047,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,4095,0.06090133388837179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,4095,0.04373333354791006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1,0.008922666932145754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,3,0.00873066671192646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,7,0.008896000062425932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,15,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,63,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,127,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,255,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,255,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,511,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,511,0.027674667537212372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1023,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1023,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,2047,0.04182933270931244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,2047,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,4095,0.06643733382225037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,4095,0.060234665870666504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,8191,0.11161599556605022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,8191,0.09993066390355428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,16383,0.2055520017941793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,16383,0.17747199535369873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1,0.008890666688481966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,7,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,7,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,63,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,127,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,255,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,511,0.02387733260790507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,511,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1023,0.029504001140594482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1023,0.03125333289305369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,2047,0.04195199906826019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,2047,0.04161600023508072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,4095,0.06612266600131989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,4095,0.06214933097362518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,8191,0.1128480037053426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,8191,0.10148266951243083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,16383,0.20577067136764526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,16383,0.17799999316533408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,15,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,31,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,127,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,511,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,511,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1023,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,2047,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,2047,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,4095,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,4095,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,8191,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,8191,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,16383,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,16383,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,3,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,3,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,7,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,7,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,15,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,15,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,31,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,31,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,63,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,63,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,127,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,127,0.020848001043001812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,255,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,255,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,511,0.023354666928450268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,511,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1023,0.03331200033426285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1023,0.0336053321758906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,2047,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,2047,0.053861334919929504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,4095,0.096778670946757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,4095,0.09090666969617207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,8191,0.17903999487559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,8191,0.1666719913482666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,16383,0.3450719912846883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,16383,0.315610667069753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,3,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,3,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,7,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,7,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,15,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,15,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,31,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,31,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,63,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,63,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,127,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,127,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,255,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,255,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,511,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,511,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1023,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1023,0.035232000052928925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,2047,0.05409066875775655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,2047,0.054042667150497437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,4095,0.09692266583442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,4095,0.09116799632708232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,8191,0.17914666732152304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,8191,0.16660267114639282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,16383,0.34491201241811115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,16383,0.3160960078239441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,7,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,63,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,127,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,255,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,255,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,511,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,2047,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,2047,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,4095,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,4095,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,8191,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,8191,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,16383,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,16383,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1,0.06760533154010773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1,0.058037335673967995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,3,0.06875200072924297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,3,0.058975999553998314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,7,0.07027733325958252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,7,0.06195733447869619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,15,0.08483200271924336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,15,0.07653333246707916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,31,0.0848586658636729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,31,0.07652799785137177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,63,0.08469333251317342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,63,0.07663999994595845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,127,0.10040533542633057
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,127,0.0929813285668691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,255,0.1504639983177185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,255,0.14205333590507507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,511,0.25058666865030926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,511,0.23933867613474527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1023,0.4524426857630412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1023,0.4322133461634318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,2047,0.85261336962382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,2047,0.8192266623179117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1,0.0683840016523997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1,0.057861333092053734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,3,0.06942933301130931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,3,0.0602453351020813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,7,0.07044266661008199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,7,0.06214400132497152
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,15,0.08524800340334575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,15,0.0767146646976471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,31,0.08475733796755473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,31,0.07624533275763194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,63,0.08482666810353597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,63,0.07657066484292348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,127,0.10102400183677673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,127,0.09286399682362874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,255,0.15057067076365152
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,255,0.14214932918548584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,511,0.2507786750793457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,511,0.23865065972010294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1023,0.4580746491750081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1023,0.4332053263982137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,2047,0.8746933142344157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,2047,0.8314826488494873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,3,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,3,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,7,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,7,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,15,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,15,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,31,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,31,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,63,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,63,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,127,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,127,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,255,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,255,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,511,0.022917332748572033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,511,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1023,0.035386666655540466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1023,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,2047,0.06108266611893972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,2047,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,3,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,3,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,7,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,15,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,15,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,31,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,31,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,63,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,63,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,127,0.02160533269246419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,127,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,255,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,255,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,511,0.03158933420976003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,511,0.028170667588710785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1023,0.04892266790072123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1023,0.04248000184694926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,2047,0.08019733428955078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,2047,0.07308800021807353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,4095,0.14229333400726318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,4095,0.1328426698843638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,8191,0.2669439911842346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,8191,0.25195733706156415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,16383,0.5164479811986288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,16383,0.48837331930796307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,3,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,3,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,7,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,15,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,15,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,31,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,31,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,63,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,63,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,127,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,127,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,255,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,511,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,511,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1023,0.04799999793370565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1023,0.042362665136655174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,2047,0.08057066798210144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,2047,0.07246399919191997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,4095,0.1421226660410563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,4095,0.1320319970448812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,8191,0.2672853271166484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,8191,0.25205334027608234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,16383,0.5600266853968302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,16383,0.49140799045562744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,511,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,511,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1023,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1023,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,2047,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,2047,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,4095,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,8191,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,8191,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,16383,0.03772799919048945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,16383,0.02699200063943863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1,0.12800000111262003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1,0.10714133580525716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,3,0.13185600439707437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,3,0.11132267117500305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,7,0.13380799690882364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,7,0.11668800314267476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,15,0.16262400150299072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,15,0.14603199561436972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,31,0.16287466883659363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,31,0.14621866742769876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,63,0.1637226641178131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,63,0.14612799882888794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,127,0.193615992863973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,127,0.17708265781402588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,255,0.2916213274002075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,255,0.2749600013097127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,511,0.4884106715520223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,511,0.4656906525293986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1023,0.8859893480936686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1023,0.8424373467763265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1,0.12799466649691263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1,0.10749333103497823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,3,0.13194666306177774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,3,0.11153599619865417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,7,0.13406399885813394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,7,0.11588266491889954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,15,0.16265066464742026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,15,0.14638400077819824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,31,0.1641706625620524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,31,0.1463093360265096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,63,0.1643786629041036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,63,0.14642666776974997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,127,0.193557341893514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,127,0.17788799603780112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,255,0.29203200340270996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,255,0.2759840091069539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,511,0.5001920064290365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,511,0.4676693280537923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1023,0.9092586835225424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1023,0.855509360631307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1,0.022128000855445862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,3,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,3,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,7,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,7,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,15,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,15,0.023061332603295643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,31,0.024586667617162068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,31,0.023034666975339253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,63,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,63,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,127,0.023434666295846302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,127,0.02330133318901062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,255,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,255,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,511,0.03789866715669632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,511,0.02922666569550832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1023,0.05611733098824819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1023,0.04181866844495138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,1,0.24893333514531454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,1,0.2059040069580078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,3,0.25674132506052655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,3,0.2146186629931132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,7,0.26079465945561725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,7,0.2262453238169352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,15,0.31860266129175824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,15,0.2834506630897522
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,31,0.31861333052317303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,31,0.28352532784144086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,63,0.32071999708811444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,63,0.28540800015131634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,127,0.37879467010498047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,127,0.34544531504313153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,255,0.5738240083058676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,255,0.537712017695109
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,1,0.2469066580136617
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,1,0.20755734046300253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,3,0.25914667050043744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,3,0.2146986722946167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,7,0.2630773385365804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,7,0.22612265745798746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,15,0.31909332672754925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,15,0.2828480005264282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,31,0.3192586700121562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,31,0.28383467594782513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,63,0.3224053382873535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,63,0.28545600175857544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,127,0.38179198900858563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,127,0.34695998827616376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,255,0.5968960126241049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,255,0.5389013290405273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,1,0.037445334096749626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,1,0.035690667728583016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,3,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,3,0.03461333364248276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,7,0.03743999948104223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,7,0.03347733368476232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,15,0.037392000357309975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,15,0.035530666510264076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,31,0.0374293327331543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,31,0.03468266626199087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,63,0.03724266588687897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,63,0.03549333413441976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,127,0.037962667644023895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,127,0.03534399966398875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,255,0.040778666734695435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,255,0.03532800078392029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,1,0.49035199483235675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,1,0.40456000963846844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,3,0.5079733530680338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,3,0.42280534903208417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,7,0.5172479947408041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,7,0.4443306525548299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,15,0.6316853364308676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,15,0.5565919876098633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,31,0.6301600138346354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,31,0.5602399905522665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,63,0.6355573336283366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,63,0.5642293294270834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,127,0.7508959770202637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,127,0.6835253238677979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,1,0.48877867062886554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,1,0.407477339108785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,3,0.5087946653366089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,3,0.42338132858276367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,7,0.5187786817550659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,7,0.4452586571375529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,15,0.6315466562906901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,15,0.5576693216959635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,31,0.6312959988911947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,31,0.559930682182312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,63,0.6381973425547282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,63,0.5627306699752808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,127,0.7852693398793539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,127,0.6916320323944092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,1,0.063509335120519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,1,0.0582826683918635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,3,0.06401066482067108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,3,0.06007466713587443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,7,0.0647680014371872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,7,0.06000000238418579
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,15,0.06421866516272227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,15,0.06003733476003011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,31,0.06434133152167003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,31,0.05960000058015188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,63,0.06460799773534139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,63,0.06020266811052958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,127,0.066021333138148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,127,0.059215997656186424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,3,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,3,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,7,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,7,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,15,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,15,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,31,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,31,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,63,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,63,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,127,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,127,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,255,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,255,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,511,0.04350399971008301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,511,0.04162666698296865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1023,0.07437333464622498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1023,0.0724426656961441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,2047,0.13568533460299173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,2047,0.13211733102798462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,4095,0.25683732827504474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,4095,0.2526506582895915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,8191,0.5026666720708212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,8191,0.49587734540303546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,16383,1.0286186536153157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,16383,1.1034506956736247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,3,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,7,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,7,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,15,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,15,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,31,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,31,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,63,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,63,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,127,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,127,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,255,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,255,0.027306665976842243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,511,0.043968002001444496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,511,0.041749333341916404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1023,0.07402666906515758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1023,0.07253866891066234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,2047,0.1346453328927358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,2047,0.13274133205413818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,4095,0.2585973342259725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,4095,0.25327465931574505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,8191,0.5049066543579102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,8191,0.49664000670115155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,16383,1.1289760271708171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,16383,1.2002986272176106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,7,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,15,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,511,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1023,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1023,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,2047,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,2047,0.017866666118303936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,4095,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,4095,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,8191,0.03870933254559835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,8191,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,16383,0.060165335734685264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,16383,0.04289066791534424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,1,0.9730400244394938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,1,0.7998080253601074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,3,1.0089279810587566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,3,0.8356213569641113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,7,1.0294666290283203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,7,0.8788320223490397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,15,1.2535520394643147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,15,1.1067519982655842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,31,1.2526559829711914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,31,1.111237366994222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,63,1.2645599842071533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,63,1.118725299835205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,1,0.9657920201619467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,1,0.8061227003733317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,3,1.0114186604817708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,3,0.8388373057047526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,7,1.036959966023763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,7,0.8823200066884359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,15,1.264965295791626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,15,1.1103413105010986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,31,1.2850293318430583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,31,1.121077299118042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,63,1.3133172988891602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,63,1.1546613375345867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,1,0.11746666828791301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,1,0.10944533348083496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,3,0.11808000008265178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,3,0.10920000076293945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,7,0.11760000387827556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,7,0.10950932900110881
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,15,0.11779200037320454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,15,0.10935466488202412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,31,0.11824533343315125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,31,0.10935466488202412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,63,0.11780266960461934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,63,0.10949866970380147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,1,2.044544061024984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,1,1.6755146980285645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,3,2.1003519694010415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,3,1.7244213422139485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,7,2.135120073954264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,7,1.8094186782836914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,15,2.5501707394917807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,15,2.253935972849528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,31,2.555957317352295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,31,2.26582940419515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,1,2.109824021657308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,1,1.739024003346761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,3,2.16866127649943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,3,1.777184009552002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,7,2.218853314717611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,7,1.858415921529134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,15,2.5866506894429526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,15,2.285893281300863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,31,2.5927573839823403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,31,2.2968533833821616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,1,0.22245866060256958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,1,0.20458134015401205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,3,0.22175999482472739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,3,0.20500266551971436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,7,0.22287466128667197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,7,0.204367995262146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,15,0.2222506602605184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,15,0.20439465840657553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,31,0.22242132822672525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,31,0.20453333854675293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,3,0.023472001155217487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,3,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,7,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,7,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,15,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,15,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,31,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,31,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,63,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,63,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,127,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,127,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,255,0.044735997915267944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,255,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,511,0.07216000060240428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,511,0.07045333087444305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1023,0.12785067160924277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1023,0.12376532951990764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,2047,0.23672000567118326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,2047,0.23241066932678223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,4095,0.45843732357025146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,4095,0.44523199399312335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,8191,0.8984213670094808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,8191,0.8777013619740804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,3,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,3,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,7,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,7,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,15,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,15,0.026021334032217663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,31,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,31,0.02569066733121872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,63,0.02779199928045273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,63,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,127,0.03147733211517334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,127,0.03014400104681651
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,255,0.045935998360315956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,255,0.044064000248909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,511,0.07301866511503856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,511,0.07097599903742473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1023,0.1281066636244456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1023,0.12425600488980611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,2047,0.23823465903600058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,2047,0.2319413423538208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,4095,0.46434664726257324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,4095,0.4474826653798421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,8191,0.9152746995290121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1,0.012335999558369318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,3,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,3,0.012175999581813812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,7,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,7,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,8191,0.8834293683369955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,15,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,15,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,31,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,31,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,63,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,255,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,511,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,511,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1023,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1023,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,2047,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,2047,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,4095,0.037471999724706016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,4095,0.02734400083621343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,8191,0.06021333237489065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,8191,0.04192000130812327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1,0.025029333929220837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,3,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,3,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,3,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,3,0.023247999449570973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,7,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,7,0.02897600084543228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,7,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,7,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,15,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,15,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,15,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,15,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,31,0.033626665671666466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,31,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,31,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,31,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,63,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,63,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,63,0.030213333666324615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,63,0.029605334003766377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,127,0.03369066615899404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,127,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,127,0.031290667752424874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,255,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,255,0.039919999738534294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,255,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,255,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,511,0.03395200024048487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,511,0.05612266560395559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,511,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,511,0.05379733443260193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1023,0.03370666752258936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1023,0.09097599983215332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1023,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1023,0.0888426701227824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,2047,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,2047,0.1607200006643931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,2047,0.03108799954255422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,2047,0.15851733088493347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,4095,0.033413333197434746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,4095,0.3016960024833679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,4095,0.03125333289305369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,4095,0.29772265752156574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,8191,0.03372266640265783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,8191,0.5849546591440836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,8191,0.031178665657838184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,8191,0.5793546835581461
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1,0.02770666778087616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1,0.027562665442625683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1,0.023418667415777843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1,0.024634666740894318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,3,0.028143999477227528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,3,0.028069332242012024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,3,0.02481599897146225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,3,0.025018667181332905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,7,0.029946667452653248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,7,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,7,0.025674665967623394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,7,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,15,0.029296000798543293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,15,0.02940266579389572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,15,0.025706666211287182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,15,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,31,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,31,0.03331200033426285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,31,0.029669334491093952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,31,0.02985599885384242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,63,0.033941333492596946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,63,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,63,0.03141333411137263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,63,0.03142400085926056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,127,0.03400533397992452
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,127,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,127,0.03130666663249334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,127,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,255,0.03340800106525421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,255,0.03984000037113825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,255,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,255,0.0356480007370313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,511,0.03349866718053818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,511,0.05934933324654897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,511,0.05606933434804281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,511,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,511,0.05425600210825602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1023,0.03355200091997782
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1023,0.09110933542251587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1023,0.031290667752424874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1023,0.08866666754086812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,2047,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,2047,0.16248533129692078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,2047,0.031386665999889374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,2047,0.15869333346684775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,4095,0.03374933451414108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,4095,0.031397332747777305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,4095,0.30210665861765545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,4095,0.2980586687723796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,8191,0.03346666693687439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,8191,0.03125333289305369
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,8191,0.5930773417154948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,8191,0.5785653193791708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1,0.012479999413092932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,7,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,15,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,15,0.01239466667175293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,31,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,63,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,63,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,511,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,511,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,511,0.01246400053302447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1023,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1023,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1023,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1023,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,2047,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,2047,0.02054399996995926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,2047,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,2047,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,4095,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,4095,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,4095,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,4095,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,8191,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,8191,0.0430026650428772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,8191,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,8191,0.03562133262554804
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,3,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,3,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,3,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,3,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,7,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,7,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,7,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,7,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,15,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,15,0.014405333747466406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,15,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,15,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,31,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,31,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,31,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,31,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,63,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,63,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,63,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,127,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,127,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,127,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,127,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,255,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,255,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,255,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,255,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,511,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,511,0.014639999717473984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,511,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1023,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1023,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1023,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1023,0.016208000481128693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,2047,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,2047,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,2047,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,2047,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,4095,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,4095,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,4095,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,4095,0.033626665671666466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,8191,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,8191,0.062133332093556724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,8191,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1,0.009072000160813332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,8191,0.05470400055249532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,3,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,3,0.00879466657837232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,3,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,7,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,7,0.008826666822036108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,7,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,15,0.009285333255926767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,15,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,31,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,31,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,127,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,127,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,255,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,255,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,255,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,511,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,511,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1023,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1023,0.021525333325068157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1023,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1023,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,2047,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,2047,0.027162666122118633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,2047,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,2047,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,4095,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,4095,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,4095,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,4095,0.03835200021664301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,8191,0.019834666202465694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,8191,0.05026666820049286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,8191,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,8191,0.05050666630268097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,16383,0.01964266722400983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,16383,0.08083733419577281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,16383,0.023050665855407715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,16383,0.07683733105659485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,32767,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,32767,0.1423306663831075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,32767,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,32767,0.12799466649691263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1,0.009317333499590555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,3,0.008986666798591614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,3,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,7,0.009312000125646591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,7,0.00927466650803884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,15,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,15,0.008965333302815756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,15,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,31,0.00919999989370505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,31,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,127,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,255,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,511,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,511,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1023,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1023,0.021909333765506744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1023,0.015573333948850632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1023,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,2047,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,2047,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,2047,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,2047,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,4095,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,4095,0.03350933392842611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,4095,0.02325333406527837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,4095,0.037615999579429626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,8191,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,8191,0.050240000089009605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,8191,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,8191,0.04982399940490723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,16383,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,16383,0.08057599763075511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,16383,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,16383,0.07685866455237071
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,32767,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,32767,0.14221866925557455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,32767,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,32767,0.1279306709766388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,3,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,7,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,15,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,31,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,31,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,127,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,255,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,255,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,255,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,255,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,511,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,511,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,511,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,511,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1023,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1023,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1023,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,2047,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,2047,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,2047,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,2047,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,4095,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,4095,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,4095,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,4095,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,8191,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,8191,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,8191,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,8191,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,16383,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,16383,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,16383,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,16383,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,32767,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,32767,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,32767,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,32767,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,7,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,15,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,15,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,31,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,255,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,511,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1023,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1023,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1023,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,2047,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,2047,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,2047,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,2047,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,4095,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,4095,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,4095,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,4095,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,8191,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,8191,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,8191,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,8191,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,16383,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,16383,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,16383,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,16383,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,32767,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,32767,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,32767,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,32767,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1,0.009098666409651438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1,0.00914666677514712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,3,0.009477333476146063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,3,0.009690666571259499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,3,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,7,0.010026666646202406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,7,0.0099093330403169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,15,0.009754666437705358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,15,0.009370666618148485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,15,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,31,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,31,0.009109333157539368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,31,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,63,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,127,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,255,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,255,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,255,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,511,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,511,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1023,0.019765333582957584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1023,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1023,0.024911999702453613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1023,0.028805332879225414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,2047,0.01945066700379054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,2047,0.031130666534105938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,2047,0.02499733368555705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,2047,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,4095,0.02089600016673406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,4095,0.042021334171295166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,4095,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,4095,0.043712000052134194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,8191,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,8191,0.06654933094978333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,8191,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,8191,0.06246933341026306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,16383,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,16383,0.11162133018175761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,16383,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,16383,0.10130666693051656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,32767,0.020853333175182343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,32767,0.20580265919367471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,32767,0.024832000335057575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,32767,0.17881600062052408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,3,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,3,0.008832000195980072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,7,0.009343999748428663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,7,0.009658666948477427
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,15,0.00961599995692571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,15,0.009461333354314169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,31,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,31,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,127,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,127,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,127,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,255,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,511,0.02075200031201045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,511,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,511,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,511,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1023,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1023,0.02479466547568639
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1023,0.02475200096766154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1023,0.029157333076000214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,2047,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,2047,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,2047,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,2047,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,4095,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,4095,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,4095,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,4095,0.043706665436426796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,8191,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,8191,0.06668266654014587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,8191,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,8191,0.06284266710281372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,16383,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,16383,0.11181867122650146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,16383,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,16383,0.1016533374786377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,32767,0.020869334538777668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,32767,0.20577067136764526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,32767,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,32767,0.18002132574717203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,7,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,15,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,15,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,31,0.013552000125249227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,127,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,255,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,255,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,511,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1023,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,2047,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,2047,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,2047,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,2047,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,4095,0.01211200033624967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,4095,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,4095,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,4095,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,8191,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,8191,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,8191,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,8191,0.016565332810084026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,16383,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,16383,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,16383,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,16383,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,32767,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,32767,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,32767,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,32767,0.020538666596015293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,7,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,15,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,127,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,127,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,255,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,511,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,511,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,511,0.011616000284751257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1023,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1023,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,2047,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,2047,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,4095,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,4095,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,4095,0.012202666451533636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,4095,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,8191,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,8191,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,8191,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,8191,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,16383,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,16383,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,16383,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,16383,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,32767,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,32767,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,32767,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,32767,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1,0.0481279989083608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1,0.048101335763931274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1,0.04027733455101649
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,3,0.04794133206208547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,3,0.047775998711586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,3,0.039749334255854286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,3,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,7,0.05011733373006185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,7,0.0497920016447703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,1,0.09922666351000468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,7,0.04192533095677694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,7,0.04186666508515676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,15,0.050383999943733215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,15,0.050101334849993386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,15,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,15,0.0436106671889623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,31,0.0598826656738917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,31,0.060080001751581825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,31,0.05398933092753092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,31,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,63,0.06033066908518473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,63,0.06018133461475372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,63,0.053930665055910744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,63,0.05376000205675761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,127,0.06018133461475372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,127,0.0605973352988561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,127,0.05426666637261709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,127,0.05411200225353241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,255,0.06011199951171875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,255,0.07062933345635732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,255,0.0539680023988088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,255,0.06438399851322174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,511,0.060271998246510826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,511,0.10379733641942342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,511,0.05388799806435903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,511,0.09738666812578838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1023,0.060266668597857155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1023,0.0539680023988088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1023,0.16293866435686746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,2047,0.06025066475073496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,2047,0.30408533414204914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,2047,0.05417599777380625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,2047,0.2961066762606303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,4095,0.06067200005054474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,4095,0.57259734471639
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,4095,0.05369600156943003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,4095,0.5627466837565104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1,0.04795733094215393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1,0.04780800143877665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1,0.04010133445262909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,3,0.04788800080617269
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,3,0.047770669062932335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,3,0.03989866624275843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,3,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,7,0.049957334995269775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,7,0.04987733562787374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,7,0.041759997606277466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,7,0.04155199974775314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,15,0.050069332122802734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,15,0.04990399877230326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,15,0.043840001026789345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,15,0.043605332573254905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,31,0.06001600126425425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,31,0.05996266504128774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,31,0.05328000088532766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,31,0.05415999889373779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,63,0.06131199995676676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,63,0.06043200194835663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,63,0.05428266525268555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,63,0.05439466734727224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,127,0.060831998785336815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,127,0.060831998785336815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,127,0.05397333204746246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,127,0.05406933526198069
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,255,0.06087466577688853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,255,0.0707946668068568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,255,0.05379733443260193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,255,0.06437866886456807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,511,0.06031466523806254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,511,0.103493332862854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,511,0.054048001766204834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,511,0.09729599952697754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1023,0.06032533446947733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1023,0.16917866468429565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1023,0.05403199791908264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1023,0.16314666469891867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,2047,0.06035733222961426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,2047,0.30401066939036053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,2047,0.05395199855168661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,2047,0.29598400990168255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,4095,0.06052800019582113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,4095,0.5767413377761841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,4095,0.05412800113360087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,4095,0.5633973280588785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,3,0.01402666668097178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,3,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,3,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,3,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,7,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,7,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,7,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,7,0.013663999736309052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,15,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,15,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,15,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,31,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,31,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,31,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,31,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,63,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,63,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,63,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,63,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,127,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,127,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,127,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,127,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,255,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,255,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,255,0.014208000153303146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,255,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,511,0.013525333255529404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,511,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,511,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,511,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1023,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1023,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1023,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,2047,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,2047,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,2047,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,2047,0.026757332185904186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,4095,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,4095,0.04379733403523763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,4095,0.0373279998699824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,3,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,3,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,3,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,3,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,7,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,7,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,7,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,7,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,15,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,15,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,15,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,15,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,31,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,31,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,31,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,31,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,63,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,63,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,63,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,63,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,127,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,127,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,127,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,127,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,255,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,255,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,255,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,255,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,511,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,511,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,511,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,511,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1023,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1023,0.02489600082238515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1023,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1023,0.02179199953873952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,2047,0.018298666924238205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,2047,0.0405173326532046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,2047,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,2047,0.035599999129772186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,4095,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,4095,0.06228266656398773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,4095,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,4095,0.05595199763774872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,3,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,3,0.014432000617186228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,7,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,7,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,7,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,15,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,15,0.01469333345691363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,15,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,31,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,31,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,63,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,63,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,63,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,63,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,127,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,127,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,127,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,255,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,255,0.016805333395799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,255,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,255,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,511,0.016773333152135212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,511,0.018288000176350277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,511,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1023,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1023,0.023525332411130268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1023,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1023,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,2047,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,2047,0.0352960005402565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,2047,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,2047,0.031248000760873158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,4095,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,4095,0.056101332108179726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,4095,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,4095,0.05165866514046987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,8191,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,8191,0.09735999504725139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,8191,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,8191,0.09114133318265279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,16383,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,16383,0.18082133928934732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,16383,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,16383,0.17308266957600912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,32767,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,32767,0.3473493258158366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,32767,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,32767,0.3509226640065511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,3,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,3,0.014698666830857595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,3,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,7,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,7,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,15,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,15,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,31,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,31,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,31,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,31,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,63,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,63,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,63,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,63,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,127,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,127,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,127,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,127,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,255,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,255,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,255,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,255,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,511,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,511,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,511,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,511,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1023,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1023,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1023,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1023,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,2047,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,2047,0.03509333233038584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,2047,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,2047,0.031290667752424874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,4095,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,4095,0.056218668818473816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,4095,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,4095,0.04981866478919983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,8191,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,8191,0.09718400239944458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,8191,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,8191,0.09129599730173747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,16383,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,16383,0.18094933032989502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,16383,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,16383,0.17284266153971353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,32767,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,32767,0.3473546504974365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,32767,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,32767,0.3508373498916626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,255,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,511,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1023,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,2047,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,2047,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,2047,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,4095,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,4095,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,4095,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,4095,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,8191,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,8191,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,8191,0.018618666877349217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,16383,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,16383,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,16383,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,16383,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,32767,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,32767,0.029472000896930695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,32767,0.01806933308641116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,32767,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,3,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,3,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,3,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,3,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,15,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,31,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,31,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,127,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,255,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,255,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,511,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,511,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1023,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,2047,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,2047,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,4095,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,4095,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,4095,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,8191,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,8191,0.01966933285196622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,8191,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,8191,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,16383,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,16383,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,16383,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,16383,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,32767,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,32767,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,32767,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,32767,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1,0.08906666437784831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1,0.08940266569455464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1,0.07441600163777669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1,0.0745600014925003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,3,0.09056533376375835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,3,0.09020266930262248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,3,0.07432533303896587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,3,0.07444266478220622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,7,0.09276266892751057
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,7,0.09282132983207703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,7,0.0766133318344752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,7,0.07644799848397572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,15,0.09361599882443745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,15,0.09317866961161296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,15,0.0788373351097107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,15,0.07884266475836436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,31,0.11150933305422465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,31,0.11349333326021831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,31,0.09912000099817912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,31,0.09938666224479675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,63,0.11349333326021831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,63,0.11339199542999268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,63,0.09940800070762634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,63,0.09954133629798889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,127,0.115365336338679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,127,0.11532266934712727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,127,0.10104533036549886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,127,0.10121599833170573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,255,0.11531733473141988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,255,0.13409066200256348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,255,0.10105599959691365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,255,0.12086932857831319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,511,0.11540266871452332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,511,0.19977599382400513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,511,0.09937600294748943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,511,0.1861813267072042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1023,0.11619200309117635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1023,0.3291520078976949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1023,0.0997279981772105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1023,0.31698666016260785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,2047,0.11555733283360799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,2047,0.6017599900563558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,2047,0.10074667135874431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,2047,0.5821280082066854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1,0.08911466598510742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1,0.08965866764386494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1,0.0743999977906545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1,0.07906133433183034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,3,0.09687999884287517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,3,0.09681600332260132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,3,0.0738613357146581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,3,0.07437333464622498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,7,0.09225066502888997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,7,0.09213866790135701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,7,0.07662400106589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,7,0.07660800218582153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,15,0.09321600198745728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,15,0.09342400232950847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,15,0.07874666651089986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,15,0.07905599971612294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,31,0.11184533437093098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,31,0.11236799756685893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,31,0.09893332918485005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,31,0.09923733274141948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,63,0.11365333199501038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,63,0.11385066310564677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,63,0.09948266545931499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,63,0.09931199749310811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,127,0.11547199885050456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,127,0.11554666360219319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,127,0.10106133421262105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,127,0.10110933581988017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,255,0.11592533191045125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,255,0.13386133313179016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,255,0.09946133693059285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,255,0.11988799770673116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,511,0.11534933249155681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,511,0.19962133963902792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,511,0.09935466448465984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,511,0.1852160096168518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1023,0.11541333794593811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1023,0.3306186596552531
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1023,0.10129599769910176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1023,0.31676799058914185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,2047,0.11555733283360799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,2047,0.6089013417561849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,2047,0.09993599851926167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,2047,0.5825386842091879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,3,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,3,0.016554666062196095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,3,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,3,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,7,0.01674666628241539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,7,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,7,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,7,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,15,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,15,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,15,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,15,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,31,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,31,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,31,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,31,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,63,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,63,0.016602666427691776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,63,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,63,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,127,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,127,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,127,0.016688000410795212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,127,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,255,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,255,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,255,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,255,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,511,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,511,0.020453333854675293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,511,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,511,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1023,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1023,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1023,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1023,0.025493333737055462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,2047,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,2047,0.04577599962552389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,2047,0.01969066634774208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,2047,0.03957866628964742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1,0.019658666104078293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,3,0.019733333339293797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,3,0.019472000499566395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,3,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,3,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,7,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,7,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,7,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,7,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,15,0.01959466685851415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,15,0.01951466624935468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,15,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,15,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,31,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,31,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,31,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,31,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,63,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,63,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,63,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,63,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,127,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,127,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,127,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,127,0.01941866676012675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,255,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,255,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,255,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,255,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,511,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,511,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,511,0.01950399950146675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,511,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1023,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1023,0.03738666574160258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1023,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1023,0.03176533430814743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,2047,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,2047,0.05860800047715505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,2047,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,2047,0.05285866558551788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,1,0.17106133699417114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,1,0.17255999644597372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,1,0.13963199655214945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,1,0.13984533150990805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,3,0.17234132687250772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,3,0.17203734318415323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,3,0.13983999689420065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,3,0.13962666193644205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,7,0.17702933152516684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,7,0.17675199111302695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,7,0.144378662109375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,7,0.14421332875887552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,15,0.18088533480962118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,15,0.18118399381637573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,15,0.1504586637020111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,15,0.15043733517328897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,31,0.21782932678858438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,31,0.21779199441274008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,31,0.1909066637357076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,31,0.19125332434972128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,63,0.21997867027918497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,63,0.22016000747680664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,63,0.19153600931167603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,63,0.19196800390879312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,127,0.22194133202234903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,127,0.22191466887791952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,127,0.19353600343068442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,127,0.19329599539438883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,255,0.22203733523686728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,255,0.26090667645136517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,255,0.19343467553456625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,255,0.2336426575978597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,511,0.22222399711608887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,511,0.3900373379389445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,511,0.19339734315872192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,511,0.36530133088429767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,1,0.17114667097727457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,1,0.1714293360710144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,1,0.14620799819628397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,1,0.14589333534240723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,3,0.17273600896199545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,3,0.17087467511494955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,3,0.1400320033232371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,3,0.13970133662223816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,7,0.17704532543818155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,7,0.17705599466959634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,7,0.14453333616256714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,7,0.14414933323860168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,15,0.18099733193715414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,15,0.18095467487970987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,15,0.15039466818173727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,15,0.15052800377209982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,31,0.21809599796930948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,31,0.21789334217707315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,31,0.1912320057551066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,31,0.19150932629903158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,63,0.220143993695577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,63,0.22033067544301352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,63,0.19305066267649332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,63,0.19405867656071982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,127,0.22262932856877646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,127,0.22220800320307413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,127,0.19396267334620157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,127,0.19370132684707642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,255,0.22244799137115479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,255,0.2610666751861572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,255,0.193615992863973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,255,0.23307732741038004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,511,0.22234666347503662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,511,0.39235734939575195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,511,0.19350399573644003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,511,0.3643733263015747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,1,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,1,0.022485333184401195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,1,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,1,0.021418665846188862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,3,0.021925332645575207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,3,0.02199466774861018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,3,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,3,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,7,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,7,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,7,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,7,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,15,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,15,0.021856000026067097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,15,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,15,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,31,0.02160533269246419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,31,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,31,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,31,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,63,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,63,0.022656001150608063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,63,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,63,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,127,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,127,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,127,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,127,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,255,0.022175999979178112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,255,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,255,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,255,0.02165333429972331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,511,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,511,0.02937600016593933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,511,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,511,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,1,0.027658666173617046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,1,0.02757333219051361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,1,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,1,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,3,0.02777066578467687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,3,0.029130667448043823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,3,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,3,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,7,0.029114666084448498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,7,0.029359998802344005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,7,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,7,0.027301333844661713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,15,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,15,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,15,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,15,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,31,0.027647999425729115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,31,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,31,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,31,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,63,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,63,0.02932800104220708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,63,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,63,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,127,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,127,0.027424000203609467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,127,0.027045334378878277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,127,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,255,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,255,0.029130667448043823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,255,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,255,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,511,0.029167999823888142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,511,0.04207466542720795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,511,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,511,0.03735466549793879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,1,0.3365279833475749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,1,0.3359839916229248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,1,0.2696320017178853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,1,0.26999467611312866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,3,0.3362506628036499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,3,0.33640531698862713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,3,0.26949866612752277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,3,0.26976533730824787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,7,0.3470453421274821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,7,0.34700266520182294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,7,0.28141866127649945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,7,0.28174400329589844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,15,0.35360534985860187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,15,0.35339732964833576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,15,0.2937440077463786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,15,0.29422932863235474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,31,0.4293813308080037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,31,0.42925333976745605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,31,0.3740053176879883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,31,0.37429332733154297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,63,0.4330506722132365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,63,0.43320000171661377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,63,0.3778293530146281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,63,0.3776586850484212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,127,0.43537600835164386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,127,0.4362026850382487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,127,0.37988801797231037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,127,0.4121546745300293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,255,0.4368799924850464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,255,0.5130506753921509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,255,0.3794613281885783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,255,0.4595786730448405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,1,0.3348426818847656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,1,0.3617493311564128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,1,0.2711840073267619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,1,0.2714560031890869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,3,0.3349119822184245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,3,0.3352320194244385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,3,0.2712480028470357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,3,0.27113600571950275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,7,0.3468799988428752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,7,0.3469546635945638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,7,0.28149332602818805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,7,0.281605343023936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,15,0.3526719808578491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,15,0.353221337000529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,15,0.2937653263409932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,15,0.2938399910926819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,31,0.4285600185394287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,31,0.4285653432210286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,31,0.37357866764068604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,31,0.3737866481145223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,63,0.43303998311360675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,63,0.43254931767781574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,63,0.37785065174102783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,63,0.3779626687367757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,127,0.4353119929631551
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,127,0.4353119929631551
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,127,0.3917119900385539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,127,0.3794026772181193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,255,0.43693868319193524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,255,0.5126026471455892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,255,0.379861315091451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,255,0.4594879945119222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,1,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,1,0.03385066737731298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,1,0.035589332381884255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,1,0.03339199970165888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,3,0.033887999753157295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,3,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,3,0.031498665610949196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,3,0.031658666829268135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,7,0.03365333378314972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,7,0.03374933451414108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,7,0.033573334415753685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,7,0.03331200033426285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,15,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,15,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,15,0.03372266640265783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,15,0.03276266654332479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,31,0.033802665770053864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,31,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,31,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,31,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,63,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,63,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,63,0.0335359995563825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,63,0.033226666351159416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,127,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,127,0.033520000676314034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,127,0.0316746657093366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,127,0.03166933357715607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,255,0.03357866654793421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,255,0.033600000043710075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,255,0.033402666449546814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,255,0.0316746657093366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,1,0.04576000074545542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,1,0.046015997727712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,1,0.04379733403523763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,1,0.04374399781227112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,3,0.0459199994802475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,3,0.04619733492533366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,3,0.04394133388996124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,3,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,7,0.04606399933497111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,7,0.04606399933497111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,7,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,7,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,15,0.04587733248869578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,15,0.04561600089073181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,15,0.043578664461771645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,15,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,31,0.04587733248869578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,31,0.04613866905371348
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,31,0.043578664461771645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,31,0.04436799883842468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,63,0.04618666569391886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,63,0.04599999884764353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,63,0.043712000052134194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,63,0.043605332573254905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,127,0.046240001916885376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,127,0.046207999189694725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,127,0.04382933179537455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,127,0.04385066529115041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,255,0.04811733464399973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,255,0.04821333289146423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,255,0.04545066754023234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,255,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,3,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,3,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,3,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,3,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,7,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,7,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,7,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,7,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,15,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,15,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,15,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,15,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,31,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,31,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,31,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,31,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,63,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,63,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,63,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,63,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,127,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,127,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,127,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,127,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,255,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,255,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,255,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,255,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,511,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,511,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,511,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,511,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1023,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1023,0.0330826664964358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1023,0.020746666938066483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1023,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,2047,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,2047,0.054197331269582115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,2047,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,2047,0.05442666510740916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,4095,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,4095,0.09699733058611552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,4095,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,4095,0.09320533275604248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,8191,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,8191,0.18132267395655313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,8191,0.02083733429511388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,8191,0.17300266027450562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,16383,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,16383,0.3568640152613322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,16383,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,16383,0.329855998357137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,32767,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,32767,0.9465386867523193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,32767,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,32767,0.6888426939646403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1,0.013503999759753546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,3,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,3,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,3,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,3,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,7,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,7,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,7,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,7,0.015967999895413715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,15,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,15,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,15,0.016069332758585613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,15,0.016501333564519882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,31,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,31,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,31,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,31,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,63,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,63,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,63,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,63,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,127,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,127,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,127,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,127,0.022064000368118286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,255,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,255,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,255,0.021701333423455555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,255,0.021562665700912476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,511,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,511,0.021695998807748158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,511,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,511,0.023792001108328503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1023,0.01563199982047081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1023,0.033471999069054924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1023,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1023,0.03381866713364919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,2047,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,2047,0.05421333511670431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,2047,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,2047,0.05412266651789347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,4095,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,4095,0.09712533156077068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,4095,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,4095,0.09316266576449077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,8191,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,8191,0.18126400311787924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,8191,0.021040000021457672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,8191,0.17144532998402914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,16383,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,16383,0.3572373390197754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,16383,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,16383,0.3302453358968099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,32767,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,32767,1.0286293029785156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,32767,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,32767,0.6888960202534994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,3,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,7,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,511,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,511,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,511,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1023,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,2047,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,2047,0.01568000018596649
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,2047,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,2047,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,4095,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,4095,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,4095,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,8191,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,8191,0.02086399992307027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,8191,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,8191,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,16383,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,16383,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,16383,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,16383,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,32767,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,32767,0.044031997521718345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,32767,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,32767,0.035642666121323906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,3,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,7,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,15,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,31,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,511,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1023,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1023,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,2047,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,2047,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,2047,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,4095,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,4095,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,4095,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,4095,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,8191,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,8191,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,8191,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,8191,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,16383,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,16383,0.03961066653331121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,16383,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,16383,0.03179199993610382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,32767,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,32767,0.06272000074386597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,32767,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,32767,0.053914666175842285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,1,0.6646186510721842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,1,0.6641706625620524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,1,0.5312000115712484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,1,0.5313066641489664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,3,0.6641066471735636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,3,0.6645493507385254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,3,0.5309546788533529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,3,0.5313813289006551
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,7,0.6852586269378662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,7,0.6858987013498942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,7,0.5558453400929769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,7,0.5557173490524292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,15,0.6996800104777018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,15,0.6993652979532877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,15,0.5799573262532552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,15,0.5800960063934326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,31,0.8490400314331055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,31,0.8496906757354736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,31,0.7407733599344889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,31,0.740768035252889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,63,0.8585279782613119
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,63,0.8578879833221436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,63,0.7858986854553223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,63,0.7724800109863281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,127,0.8643360137939453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,127,0.8983200391133627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,127,0.7511680126190186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,127,0.7510133584340414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,1,0.7149279912312826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,1,0.6625973383585612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,1,0.534880002339681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,1,0.5348426500956217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,3,0.6641279856363932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,3,0.6640640099843343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,3,0.535103996594747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,3,0.5351359844207764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,7,0.6871306896209717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,7,0.6867626508076986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,7,0.5563146670659384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,7,0.5562666654586792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,15,0.69923202196757
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,15,0.6981333096822103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,15,0.5803999900817871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,15,0.5802506605784098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,31,0.8493119875590006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,31,0.8742720286051432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,31,0.7405386765797933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,31,0.7408320109049479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,63,0.8580906391143799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,63,0.8573386669158936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,63,0.7482133706410726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,63,0.747973362604777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,127,0.9234293301900228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,127,0.8678239981333414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,127,0.8155413468678793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,127,0.7510879834493002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,1,0.05749333401521047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,1,0.0580320010582606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,1,0.05413866539796194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,1,0.05435733497142792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,3,0.05806399881839752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,3,0.05667200187842051
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,3,0.05407999952634176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,3,0.05385600030422211
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,7,0.05810666580994924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,7,0.05792533357938131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,7,0.054330666859944664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,7,0.054005334774653115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,15,0.058176000912984215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,15,0.057914664347966514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,15,0.054325332244237266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,15,0.05435200035572052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,31,0.05769599974155426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,31,0.05639466643333435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,31,0.05397333204746246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,31,0.054144000013669334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,63,0.05706666906674703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,63,0.05723733206590017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,63,0.05392000079154968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,63,0.054245332876841225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,127,0.058362667759259544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,127,0.05898133416970571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,127,0.05421866476535797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,127,0.05426133175690969
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,1,0.08321066697438557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,1,0.08320533235867818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,1,0.07678933441638947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,1,0.07696000238259633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,3,0.08276799817879994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,3,0.082805335521698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,3,0.07649066547552745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,3,0.07675200204054515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,7,0.0830026666323344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,7,0.08283199866612752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,7,0.07644799848397572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,7,0.07658133407433827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,15,0.08293866614500682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,15,0.08288000027338664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,15,0.07637866834799449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,15,0.07644266883532207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,31,0.08259200056393941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,31,0.08239999910195668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,31,0.07648533085982005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,31,0.07679999868075053
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,63,0.08277866741021474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,63,0.08345599969228108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,63,0.07709866762161255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,63,0.07710400223731995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,127,0.08479467034339905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,127,0.085125337044398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,127,0.07659199833869934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,127,0.07702399790287018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,1,1.3198346296946208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,1,1.3202239672342937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,1,1.0536426703135173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,1,1.0536586443583171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,3,1.319930632909139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,3,1.3200213114420574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,3,1.0529119968414307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,3,1.052181323369344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,7,1.363317330678304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,7,1.3645493189493816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,7,1.103386640548706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,7,1.1026346683502197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,15,1.3908586502075195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,15,1.3919946352640789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,15,1.1517120202382405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,15,1.150426705678304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,31,1.691813309987386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,31,1.6925066312154133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,31,1.4725386301676433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,31,1.4730772972106934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,63,1.7101492881774902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,63,1.7091412544250488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,127,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,63,1.487610658009847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,63,1.4884319305419922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,1,1.3191946347554524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,1,1.3183146317799885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,1,1.060480038324992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,1,1.0601119995117188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,3,1.3187839984893799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,3,1.3189600308736165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,3,1.0609866778055828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,3,1.0603573322296143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,7,1.3643360137939453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,7,1.3643733660380046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,7,1.10481063524882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,7,1.1046240329742432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,15,1.3880640665690105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,15,1.3889546394348145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,15,1.153978665669759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,15,1.1533599694569905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,31,1.6946880022684734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,31,1.6940639813741047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,31,1.4740959803263347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,31,1.4745012919108074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,63,1.7338080406188965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,63,1.7316640218098958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,63,1.4883146286010742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,63,1.4890613555908203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,1,0.10534399747848511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,1,0.1050986647605896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,1,0.09918933113416036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,3,0.10518933335940044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,511,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,3,0.1053706705570221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,3,0.09830400347709656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,3,0.09756799538930257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,7,0.10377599795659383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,7,0.10549333691596985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,7,0.0983733336130778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,1,0.09967999656995137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,7,0.0974666674931844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,15,0.10531199971834819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,15,0.10575466354688008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,15,0.0981280008951823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,15,0.09745066364606221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,31,0.10521599650382996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,31,0.10460266470909119
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,31,0.09894399841626485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,31,0.09861866633097331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,63,0.105103999376297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,63,0.10410666465759277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,63,0.09909866253534953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,1,0.15448000033696493
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,63,0.09894399841626485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,1,0.15477333466211954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,1,0.14425599575042725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,1,0.14415466785430908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,3,0.15449600418408713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,3,0.14447466532389322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,3,0.15414933363596597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,3,0.14414933323860168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,7,0.1544319987297058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,7,0.1544533371925354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,7,0.14306666453679404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,7,0.14286933342615762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,15,0.15461867054303488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,15,0.15426666537920633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,15,0.14418133099873862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,15,0.14421866337458292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,31,0.15460800131162009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,31,0.15496533115704855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,31,0.14426133036613464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,31,0.14335466424624124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,63,0.15439466635386148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,63,0.14389866590499878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1,0.019402666638294857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,63,0.14426666498184204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,3,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,3,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,3,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,3,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,7,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,7,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,7,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,15,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,15,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,15,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,15,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,31,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,31,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,31,0.01814933369557063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,31,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,63,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,63,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,63,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,63,0.017717332889636356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,127,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,127,0.020975999534130096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,127,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,127,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,255,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,255,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,255,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,255,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,511,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,511,0.032842665910720825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,511,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,511,0.03138133386770884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,63,0.15467733144760132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1023,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1023,0.050293331344922386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1023,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1023,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,2047,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,2047,0.08563199639320374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,2047,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,2047,0.08474133412043254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,4095,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,4095,0.15848533312479654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,4095,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,4095,0.15586666266123453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,8191,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,8191,0.30218666791915894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,8191,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,8191,0.30081067482630414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,16383,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,16383,0.622709314028422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,16383,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1,0.019706666469573975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1,0.019578666736682255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,16383,0.6296639839808146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1,0.01584533353646596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,3,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,3,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,3,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,3,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,7,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,7,0.01956266661485036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,7,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,7,0.015696000307798386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,15,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,15,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,15,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,15,0.01573333392540614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,31,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,31,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,31,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,31,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,63,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,63,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,63,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,63,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,127,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,127,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,127,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,127,0.018197332819302876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,255,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,255,0.02309333284695943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,255,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,255,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,511,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,511,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,511,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,511,0.031136001149813335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1023,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1023,0.050160000721613564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1023,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1023,0.04790399968624115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,2047,0.02141333371400833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,2047,0.08593599994977315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,2047,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,2047,0.0849120020866394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,4095,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,4095,0.1585493286450704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,4095,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,4095,0.15659200151761374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,8191,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,8191,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,8191,0.30188266436258954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,8191,0.30154667297999066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,16383,0.662997325261434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,16383,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,16383,0.6298986673355103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,3,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,7,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,31,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,127,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,255,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,255,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,255,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,511,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,511,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,511,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1023,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1023,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1023,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,2047,0.017488000293572743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,2047,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,2047,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,4095,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,4095,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,4095,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,4095,0.020608000457286835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,8191,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,8191,0.027647999425729115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,8191,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,8191,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,16383,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,16383,0.04382933179537455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,16383,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,16383,0.03541333228349686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,7,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,15,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,31,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,127,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1023,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1023,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,2047,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,2047,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,2047,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,2047,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,4095,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,4095,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,4095,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,4095,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,8191,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,8191,0.035717333356539406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,8191,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,8191,0.03161599983771642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,16383,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,16383,0.06038400034109751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,16383,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,16383,0.05083733300367991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1,0.02349333216746648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,3,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,3,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,7,0.027653334041436512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,15,0.033344000577926636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,15,0.029696000119050343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,31,0.033370666205883026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,31,0.029626667499542236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,63,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,63,0.029605334003766377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,127,0.03917866696914037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,127,0.03577066709597906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,255,0.05598400036493937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,255,0.05454400181770325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,511,0.09132267038027446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,511,0.08794132868448894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1023,0.16157333056131998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1023,0.15627200404802957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,2047,0.3009546597798665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,2047,0.29201600948969525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,4095,0.5793706576029459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,4095,0.5612906614939371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,8191,1.146239995956421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,8191,1.1100213527679443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,3,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,3,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,7,0.028165332973003387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,7,0.02550933261712392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,15,0.03379733363787333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,15,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,31,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,31,0.029674666623274486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,63,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,63,0.030949334303538006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,127,0.03908266623814901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,127,0.035743998984495796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,255,0.05589333176612854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,255,0.053957333167394005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,511,0.09089066584904988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,511,0.08878933389981587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1023,0.16064533591270447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1023,0.15668800473213196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,2047,0.3001226584116618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,2047,0.2914186716079712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,4095,0.5861119826634725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,4095,0.5609546502431234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,8191,1.1688906351725261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,8191,1.1274133523305256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,3,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,7,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,7,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,15,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,15,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,31,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,31,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,63,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,63,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,511,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1023,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1023,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,2047,0.02149333308140437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,2047,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,4095,0.037434667348861694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,4095,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,8191,0.05931200087070465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,8191,0.042175998290379844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,3,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,3,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,7,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,7,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,15,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,15,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,31,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,31,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,63,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,127,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,255,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,255,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,511,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,511,0.014373333503802618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1023,0.019861333072185516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1023,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,2047,0.03938133269548416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,2047,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,4095,0.06044266621271769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,4095,0.03961066653331121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,8191,0.10223467151323955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,8191,0.06160533428192139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,3,0.008954666554927826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,7,0.008912000184257826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,15,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,63,0.013765333841244379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,255,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,255,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,511,0.021776000658671062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,511,0.017375999440749485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1023,0.025749333202838898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1023,0.030095999439557392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,2047,0.034186666210492454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,2047,0.03615466753641764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,4095,0.05019199848175049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,4095,0.04982399940490723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,8191,0.08112533390522003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,8191,0.0747680018345515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1023,0.170522669951121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,16383,0.14258133371671042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,32767,0.26689600944519043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,16383,0.1277653376261393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,32767,0.23010132710138956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,3,0.008736000085870424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,3,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,7,0.008863999818762144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,15,0.008725333337982496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,31,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,63,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,127,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,255,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,255,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,511,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,511,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1023,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1023,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,2047,0.034485332667827606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,2047,0.03755199909210205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,4095,0.0497920016447703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,8191,0.0810346653064092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,8191,0.07610133290290833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,16383,0.14320533474286398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,16383,0.12755733728408813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,32767,0.26706133286158246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,32767,0.2303253412246704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,3,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,63,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,255,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1023,0.012543999900420507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1023,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,2047,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,2047,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,4095,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,8191,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,8191,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,16383,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,16383,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,32767,0.022656001150608063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,32767,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1023,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1023,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,2047,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,4095,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,4095,0.015599999576807022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,8191,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,8191,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,16383,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,16383,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,32767,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,32767,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,3,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,3,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,7,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,15,0.009056000038981438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,63,0.012560000022252401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,63,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,127,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,255,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,255,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,511,0.02405333270629247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,511,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1023,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1023,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,2047,0.04195733368396759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,2047,0.042117332418759666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,4095,0.06614933411280315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,4095,0.062362665931383766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,8191,0.11370133360226949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,8191,0.10147200028101604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,16383,0.20649067560831705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,16383,0.1790613333384196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,32767,0.392192006111145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,32767,0.33477334181467694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1,0.008885333314538002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,3,0.00897066667675972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,7,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,15,0.009008000294367472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,63,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,127,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,127,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,255,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,511,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,511,0.027376001079877216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1023,0.030858665704727173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1023,0.033285332222779594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,2047,0.041759997606277466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,2047,0.04177600145339966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,4095,0.06619733572006226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,4095,0.06218666831652323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,8191,0.11334932843844096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,8191,0.10108266274134318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,16383,0.20572799444198608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,16383,0.1791306734085083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,32767,0.39164264996846515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,32767,0.33266667524973553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1,0.010415999839703241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,3,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,31,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,127,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,511,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,511,0.01232533281048139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1023,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,2047,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,4095,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,4095,0.015861333658297855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,8191,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,8191,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,16383,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,16383,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,32767,0.022944000860055287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,32767,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,127,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,511,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1023,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1023,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,2047,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,2047,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,4095,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,4095,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,8191,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,8191,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,16383,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,16383,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,32767,0.03745600084463755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,32767,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1,0.045935998360315956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,3,0.047797332207361855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,3,0.041482667128245033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,7,0.047983999053637184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,7,0.04161066561937332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,15,0.058277333776156105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,15,0.053504000107447304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,31,0.05827199916044871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,31,0.05357333521048228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,63,0.05935466786225637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,63,0.05367999772230784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,127,0.06865600248177846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,127,0.06443200012048085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,255,0.1032426655292511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,255,0.09685867031415303
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,511,0.16897066434224448
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,511,0.16110400358835855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1023,0.30405332644780475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1023,0.2904213269551595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,2047,0.5688960154851278
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,2047,0.5476853450139364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,4095,1.1126399834950764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,4095,1.062117338180542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1,0.04610133171081543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1,0.03969600051641464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,3,0.04779199759165446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,3,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,7,0.048986668388048805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,7,0.04252266883850098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,15,0.05826666454474131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,15,0.0533493310213089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,31,0.05851200222969055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,31,0.05346133311589559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,63,0.05975466469923655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,63,0.05388266841570536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,127,0.06870399912198384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,127,0.06423466900984447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,255,0.10285866260528564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,255,0.09718400239944458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,511,0.1687999963760376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,511,0.1607093314329783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1023,0.3038826584815979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1023,0.2900159955024719
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,2047,0.5771306753158569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,2047,0.5474026600519816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,4095,1.122538646062215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,4095,1.072160005569458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,3,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,3,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,7,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,7,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,15,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,15,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,31,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,31,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,63,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,63,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,127,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,255,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,255,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,511,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,511,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1023,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1023,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,2047,0.039674667020638786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,2047,0.029071999092896778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,4095,0.06215466558933258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,4095,0.0446720023949941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1,0.015674666812022526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,3,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,3,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,7,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,7,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,15,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,15,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,31,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,31,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,63,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,63,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,127,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,127,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,255,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,255,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,511,0.020074666788180668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,511,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1023,0.034474665919939675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1023,0.023658665517965954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,2047,0.06001066664854685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,2047,0.040565334260463715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,4095,0.10311999917030334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,4095,0.062234664956728615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,3,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,3,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,7,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,15,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,31,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,31,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,63,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,63,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,127,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,127,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,255,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,255,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,511,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,511,0.02080533280968666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1023,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1023,0.030421334008375805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,2047,0.05615466833114624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,2047,0.05007466673851013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,4095,0.09716799855232239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,4095,0.09076266487439473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,8191,0.18093866109848022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,8191,0.17111466328303018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,16383,0.34597333272298175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,16383,0.3328426678975423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,32767,0.6814293066660563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,32767,0.7421066761016846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,3,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,7,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,15,0.014501333236694336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,31,0.015856000284353893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,31,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,63,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,63,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,127,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,127,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,255,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,255,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,511,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,511,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1023,0.03332266708215078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1023,0.029493334392706554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,2047,0.0557226687669754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,2047,0.04997866849104563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,4095,0.09726400176684062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,4095,0.09101866682370503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,8191,0.18090667327245077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,8191,0.17097065846125284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,16383,0.3458933432896932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,16383,0.33220799763997394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,32767,0.7133119901021322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,32767,0.7404639720916748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,3,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,31,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,255,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,511,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1023,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1023,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,2047,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,2047,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,4095,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,4095,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,8191,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,8191,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,16383,0.02483733246723811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,16383,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,32767,0.037871999045213066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,32767,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,255,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1023,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,2047,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,4095,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,4095,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,8191,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,8191,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,16383,0.03756800045569738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,16383,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,32767,0.060122668743133545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,32767,0.0412266676624616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1,0.08894933263460796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1,0.07472000022729237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,3,0.090938667456309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,3,0.0766186664501826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,7,0.09225066502888997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,7,0.08059733112653096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,15,0.11147200067838033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,15,0.09920533498128255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,31,0.11148800452550252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,31,0.0993226667245229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,63,0.11141332983970642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,63,0.09947199622790019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,127,0.13169599572817484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,127,0.12034133076667786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,255,0.1973759929339091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,255,0.18492267529169717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,511,0.32838932673136395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,511,0.3135146697362264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1023,0.5970346530278524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1023,0.5667146841684977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,2047,1.1327520211537678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,2047,1.0773653189341228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1,0.0881173312664032
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1,0.0747573326031367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,3,0.09109866619110107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,3,0.07679466903209686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,7,0.09087999661763509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,7,0.08015466729799907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,15,0.11159466703732808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,15,0.09905067086219788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,31,0.11146666606267293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,31,0.09920000036557515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,63,0.1113866666952769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,63,0.09899200002352397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,127,0.13211199641227722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,127,0.12160533666610718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,255,0.19739200671513876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,255,0.1850666602452596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,511,0.32867733637491864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,511,0.3142720063527425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1023,0.6032746632893881
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1023,0.5659039815266927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,2047,1.161247968673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,2047,1.0901707013448079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,3,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,3,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,7,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,7,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,15,0.019600000232458115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,15,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,31,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,31,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,63,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,63,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,127,0.019626667102177937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,127,0.01766933376590411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,255,0.017711999515692394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,255,0.01756799966096878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,511,0.023930666347344715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,511,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1023,0.03748800108830134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1023,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,2047,0.06235733131567637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,2047,0.04383466641108195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,3,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,3,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,7,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,7,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,15,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,15,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,31,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,31,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,63,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,63,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,127,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,255,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,255,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,511,0.0352960005402565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,511,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1023,0.055061335364977516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1023,0.0356480007370313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,2047,0.09700266520182292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,2047,0.058746665716171265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,1,0.1689173380533854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,1,0.14010666807492575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,3,0.1748960018157959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,3,0.1460533340771993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,7,0.17704000075658163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,7,0.15286933382352194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,15,0.21522667010625204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,15,0.190938671429952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,31,0.2159199913342794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,31,0.1911840041478475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,63,0.2158400019009908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,63,0.1932213306427002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,127,0.25494933128356934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,127,0.2343733310699463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,255,0.3867146571477254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,255,0.36298668384552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,511,0.648202657699585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,511,0.6152533292770386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,1,0.16874667008717856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,1,0.14095466335614523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,3,0.17500799894332886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,3,0.1485973298549652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,7,0.1771626671155294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,7,0.15245333313941956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,15,0.21552000443140665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,15,0.19150400161743164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,31,0.21411732832590738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,31,0.19158399105072021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,63,0.21604265769322714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,63,0.19340799252192178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,127,0.2547946572303772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,127,0.234442671140035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,255,0.38969600200653076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,255,0.3633066813151042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,511,0.6586133241653442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,511,0.6219306786855062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,1,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,1,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,3,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,3,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,7,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,7,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,15,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,15,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,31,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,31,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,63,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,63,0.02334933231274287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,127,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,127,0.023071999351183575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,255,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,255,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,511,0.03734933336575826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,511,0.03123733401298523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,1,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,1,0.029487999776999157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,3,0.03140799949566523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,3,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,7,0.033615998923778534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,7,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,15,0.03190933416287104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,15,0.029232000311215717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,31,0.03158933420976003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,31,0.029114666084448498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,63,0.03162666658560435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,63,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,127,0.03331200033426285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,127,0.029834667841593426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,255,0.03775466730197271
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,255,0.02978666623433431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,511,0.058389330903689064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,511,0.04186133543650309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,1,0.33063467343648273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,1,0.27340267101923627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,3,0.3420693476994832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,3,0.2840106685956319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,7,0.34690133730570477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,7,0.29846932490666706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,15,0.4230560064315796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,15,0.37532798449198407
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,31,0.4225813150405884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,31,0.3758666515350342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,63,0.42666133244832355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,63,0.3780746857325236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,127,0.5030719836552938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,127,0.4596693515777588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,255,0.7630986372629801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,255,0.7162240346272787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,1,0.3290986617406209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,1,0.27498666445414227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,3,0.3429226477940877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,3,0.2855413357416789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,7,0.3477226495742798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,7,0.29865066210428876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,15,0.42314668496449787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,15,0.37566399574279785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,31,0.4230453173319499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,31,0.3757386604944865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,63,0.42667198181152344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,63,0.3787733316421509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,127,0.5046879847844442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,127,0.45956265926361084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,255,0.7863893508911133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,255,0.7179040114084879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,1,0.0395413339138031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,1,0.03695466617743174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,3,0.03986666599909464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,3,0.03740799923737844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,7,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,7,0.037471999724706016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,15,0.04101866732041041
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,15,0.03761066744724909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,31,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,31,0.03748266647259394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,63,0.039674667020638786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,63,0.03736533224582672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,127,0.04013866682847341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,127,0.03774400055408478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,255,0.042506664991378784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,255,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,1,0.05448000133037567
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,1,0.04752000172932943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,3,0.05606933434804281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,3,0.047653332352638245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,7,0.05552533268928528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,7,0.04775466521581014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,15,0.054746667544047035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,15,0.04784533381462097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,31,0.056186666091283165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,31,0.04740799963474274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,63,0.05420266588528951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,63,0.047610665361086525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,127,0.056159997979799904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,127,0.0492799977461497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,255,0.06235733131567637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,255,0.05114666620890299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,3,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,3,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,7,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,7,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,15,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,15,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,31,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,31,0.020970667401949566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,63,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,63,0.019658666104078293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,127,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,127,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,255,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,255,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,511,0.03143466760714849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,511,0.03335466732581457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1023,0.05407999952634176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1023,0.05381333331267039
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,2047,0.09644266963005066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,2047,0.09294933080673218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,4095,0.18104000886281332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,4095,0.17096533377965292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,8191,0.3495519955952962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,8191,0.32738133271535236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,16383,0.6962506771087646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,16383,0.6361226638158163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,32767,2.083797295888265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,32767,1.4605706532796223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1,0.01594666639963786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,3,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,3,0.01602666700879733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,7,0.01350933313369751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,7,0.016234666109085083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,15,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,15,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,31,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,31,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,63,0.020773333807786305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,127,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,127,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,255,0.021690666675567627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,255,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,511,0.03142933299144109
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,511,0.033610666791598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1023,0.054245332876841225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1023,0.05436266462008158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,2047,0.09663466612497966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,2047,0.09314666191736858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,4095,0.18315200010935465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,4095,0.17079466581344604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,8191,0.3539946476618449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,8191,0.32660800218582153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,16383,0.7291733423868815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,16383,0.6352320114771525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,32767,2.46342404683431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,32767,1.6803466478983562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,7,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,15,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,511,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1023,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,2047,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,2047,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,4095,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,4095,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,8191,0.023157333334287006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,8191,0.020848001043001812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,16383,0.03806400050719579
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,16383,0.02738133321205775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,32767,0.06010666489601135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,32767,0.04154666761557261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,7,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,511,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1023,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1023,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,2047,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,2047,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,4095,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,4095,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,8191,0.038047999143600464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,8191,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,16383,0.060165335734685264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,16383,0.039493332306543984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,32767,0.10140267014503479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,32767,0.060517330964406334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,1,0.6519360144933065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,1,0.5375839869181315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,3,0.6789759794871012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,3,0.562170664469401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,7,0.6899680296579996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,7,0.5909173488616943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,15,0.8394186496734619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,15,0.7405760288238525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,31,0.8406986395517985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,31,0.7436853249867758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,63,0.8450453281402588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,63,0.7506346702575684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,127,1.0094613234202068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,127,0.9108053048451742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,1,0.6501013437906901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,1,0.5458559989929199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,3,0.6789226531982422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,3,0.5634826819101969
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,7,0.691530704498291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,7,0.5913066864013672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,15,0.8401813507080078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,15,0.7437919775644938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,31,0.8421440124511719
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,31,0.7459572950998942
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,63,0.8545973300933838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,63,0.7489919662475586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,127,1.0372533003489177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,127,0.926970640818278
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,1,0.06850133339564006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,1,0.06459199885527293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,3,0.07073066631952922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,3,0.06408533453941345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,7,0.07037866612275441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,7,0.06452266871929169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,15,0.07012266914049785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,15,0.064410666624705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,31,0.07018666466077168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,31,0.06406400104363759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,63,0.07067200044790904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,63,0.06411199768384297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,127,0.0702400008837382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,127,0.06609066824118297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,1,0.09921066959698994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,1,0.08694400389989217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,3,0.09915199875831604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,3,0.0879306693871816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,7,0.09893866380055745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,7,0.08761066198348999
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,15,0.09924266735712688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,15,0.08776000142097473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,31,0.09925333658854167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,31,0.08813333511352539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,63,0.09909333785374959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,63,0.08737066388130188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,127,0.0993226667245229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,127,0.08878933389981587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,1,1.3168693383534749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,1,1.0848373572031658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,3,1.3690719604492188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,3,1.1307360331217449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,7,1.3969225883483887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,7,1.1892853577931721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,15,1.6850934028625488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,15,1.491125265757243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,31,1.6935253143310547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,31,1.498949368794759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,63,1.722442626953125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,63,1.5137866338094075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,1,1.3199893633524578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,1,1.0944159825642903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,3,1.375823974609375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,3,1.1365066369374592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,7,1.4125760396321614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,7,1.1984586715698242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,15,1.7018292744954426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,15,1.4953546524047852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,31,1.714735984802246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,31,1.5153279304504395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,63,1.7382399241129558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,63,1.542415936787923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,1,0.12597866853078207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,1,0.11801600456237793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,3,0.12601066629091898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,3,0.11782399813334148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,7,0.12593600153923035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,7,0.11822400490442912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,15,0.12584533294041952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,15,0.11866666873296101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,31,0.12570666273434958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,31,0.11788266897201538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,63,0.125408003727595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,63,0.11753599842389424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,1,0.18313066164652506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,1,0.16273066401481628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,3,0.18358933925628662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,3,0.16191466649373373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,7,0.18318400780359903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,7,0.1628426710764567
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,15,0.18331199884414673
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,15,0.16296533743540445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,31,0.18379199504852295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,31,0.16224533319473267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,63,0.18409067392349243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,63,0.16370133558909097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,3,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,3,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,7,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,7,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,15,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,15,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,31,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,31,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,63,0.02067199970285098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,63,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,127,0.023797333240509033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,127,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,255,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,255,0.030762667457262676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,511,0.05005866785844167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,511,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1023,0.08529067039489746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1023,0.08295999964078267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,2047,0.1567093332608541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,2047,0.1546453336874644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,4095,0.2998613317807515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,4095,0.2946346600850423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,8191,0.5852053165435791
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,8191,0.5773386557896932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,16383,1.3239200115203857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,16383,1.384661356608073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,3,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,3,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,7,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,7,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,15,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,15,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,31,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,31,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,63,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,63,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,127,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,127,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,255,0.03148266673088074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,255,0.029578665892283123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,511,0.049957334995269775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,511,0.04788800080617269
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1023,0.08514133095741272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1023,0.08295466502507527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,2047,0.15653866529464722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,2047,0.15454933047294617
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,4095,0.3014826575915019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,4095,0.2948213418324788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,8191,0.6135840018590292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,8191,0.5771893262863159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,16383,1.4098666508992512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,16383,1.6121226946512859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,63,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,511,0.01259200026591619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1023,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1023,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,2047,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,2047,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,4095,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,4095,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,8191,0.03950933367013931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,8191,0.028592000404993694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,16383,0.06058666606744131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,16383,0.0436106671889623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,31,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,127,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,511,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1023,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1023,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,2047,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,2047,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,4095,0.037765334049860634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,4095,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,8191,0.06030400097370148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,8191,0.03734933336575826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,16383,0.10055466492970784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,16383,0.057914664347966514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1,0.018677332748969395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,3,0.018266666680574417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,3,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,3,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,3,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,7,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,7,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,7,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,15,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,15,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,15,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,15,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,31,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,31,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,31,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,31,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,63,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,63,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,63,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,63,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,127,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,127,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,127,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,127,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,255,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,255,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,255,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,255,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,511,0.021125334004561108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,511,0.031583999594052635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,511,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,511,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1023,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1023,0.04993066688378652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1023,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1023,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,2047,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,2047,0.08679466446240743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,2047,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,2047,0.08489066362380981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,4095,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,4095,0.15821333726247153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,4095,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,4095,0.15660799543062845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,8191,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,8191,0.3019146720568339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,8191,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,8191,0.30053865909576416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,16383,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,16383,0.6686720053354899
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,16383,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,16383,0.6308000087738037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,3,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,255,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,511,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,511,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1023,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1023,0.01404800017674764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,2047,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,2047,0.017610666652520496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,2047,0.014746667196353277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,2047,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,4095,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,4095,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,4095,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,4095,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,8191,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,8191,0.027306665976842243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,8191,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,8191,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,16383,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,16383,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,16383,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,16383,0.03711466739575068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,3,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,7,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,31,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,31,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,127,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1023,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,2047,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,2047,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,2047,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,2047,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,4095,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,4095,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,4095,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,4095,0.023397333920001984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,8191,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,8191,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,8191,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,8191,0.0331839993596077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,16383,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,16383,0.0601440022389094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,16383,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,16383,0.05389333268006643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,3,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,3,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,3,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,3,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,7,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,7,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,7,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,7,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,15,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,15,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,15,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,31,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,31,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,31,0.014192000031471252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,31,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,63,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,63,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,63,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,127,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,127,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,127,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,127,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,255,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,255,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,255,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,255,0.01340266689658165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,511,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,511,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,511,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1023,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1023,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1023,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,2047,0.026634665826956432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,2047,0.015770666301250458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,2047,0.023317334552605946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,4095,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,4095,0.040021332601706185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,4095,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,4095,0.03331733246644338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,8191,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,8191,0.06235733131567637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,8191,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,8191,0.056133334835370384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,16383,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,16383,0.10525866349538167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,16383,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,16383,0.09102933605511983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1,0.00877333308259646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,3,0.008842666943868002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,3,0.00903466654320558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,3,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,7,0.009077333534757296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,7,0.009141333401203156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,7,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,15,0.008752000207702318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,15,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,31,0.00891733355820179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,31,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,63,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,63,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,127,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,127,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,127,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,255,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,255,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,511,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,511,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,511,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1023,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1023,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1023,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1023,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,2047,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,2047,0.021573332448800404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,2047,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,2047,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,4095,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,4095,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,4095,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,4095,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,8191,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,8191,0.037674665451049805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,8191,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,8191,0.03982933362325033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,16383,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,16383,0.05611200133959452
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,16383,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,16383,0.05728533367315928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,32767,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,32767,0.09241066376368205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,32767,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,32767,0.08492799599965413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,65535,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,65535,0.16677866379419962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,65535,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,65535,0.1461120049158732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,3,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,7,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,7,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,7,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,15,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,15,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,15,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,15,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,31,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,511,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,511,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,511,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,2047,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,2047,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,2047,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,2047,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,4095,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,4095,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,4095,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,4095,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,8191,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,8191,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,8191,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,8191,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,16383,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,16383,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,16383,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,16383,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,32767,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,32767,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,32767,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,32767,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,65535,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,65535,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,65535,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,65535,0.02474133421977361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,3,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,3,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,7,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,7,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,31,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,63,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,127,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,511,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,511,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,511,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1023,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1023,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1023,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,2047,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,2047,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,2047,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,2047,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,4095,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,4095,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,4095,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,4095,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,8191,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,8191,0.016751999656359356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,8191,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,8191,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,16383,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,16383,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,16383,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,16383,0.017727999637524288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,32767,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,32767,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,32767,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,32767,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,65535,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,65535,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,65535,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,65535,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,3,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,7,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,7,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,15,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,15,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,63,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,511,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1023,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1023,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,2047,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,2047,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,2047,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,2047,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,4095,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,4095,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,4095,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,4095,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,8191,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,8191,0.01685333376129468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,8191,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,8191,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,16383,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,16383,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,16383,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,16383,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,32767,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,32767,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,32767,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,32767,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,65535,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,65535,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,65535,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,65535,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1,0.008837333569924036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,3,0.00884799969693025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,3,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,3,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,7,0.008949333180983862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,7,0.008767999708652496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,15,0.008778666456540426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,15,0.009093333035707474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,15,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,31,0.009930666536092758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,31,0.009941333283980688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,63,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,63,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,63,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,127,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,127,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,255,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,511,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,511,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,511,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1023,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1023,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1023,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1023,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,2047,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,2047,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,2047,0.023455999791622162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,2047,0.029733332494894665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,4095,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,4095,0.03379733363787333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,4095,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,4095,0.03754133234421412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,8191,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,8191,0.049882665276527405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,8191,0.022944000860055287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,8191,0.04983466863632202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,16383,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,16383,0.08078399797280629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,16383,0.023045333723227184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,16383,0.0766186664501826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,32767,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,32767,0.1421280006567637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,32767,0.023455999791622162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,32767,0.1275146702925364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,65535,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,65535,0.26501333713531494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,65535,0.02290133386850357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,65535,0.2323733369509379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,3,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,7,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,15,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,31,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,255,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,511,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1023,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1023,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1023,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,2047,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,2047,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,2047,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,4095,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,4095,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,4095,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,4095,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,8191,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,8191,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,8191,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,7,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,8191,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,16383,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,16383,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,16383,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,16383,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,32767,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,32767,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,32767,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,32767,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,65535,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,65535,0.02369066576162974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,65535,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,65535,0.02348266790310542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,15,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,31,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,127,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,511,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,511,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1023,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1023,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1023,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,16383,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,2047,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,2047,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,2047,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,2047,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,4095,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,4095,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,4095,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,4095,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,8191,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,8191,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,8191,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,8191,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,16383,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,16383,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,16383,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,16383,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,32767,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,32767,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,32767,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,32767,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,65535,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,65535,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,65535,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,65535,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,3,0.010319999729593595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,3,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,7,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,31,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,63,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,127,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,127,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,255,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,255,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,511,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,511,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,2047,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,2047,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,2047,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,4095,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,4095,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,4095,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,4095,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,8191,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,8191,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,8191,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,8191,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,16383,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,16383,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,16383,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,16383,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,32767,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,32767,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,32767,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,32767,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,65535,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,65535,0.03759466608365377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,65535,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,65535,0.03152533372243246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1,0.023455999791622162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1,0.023792001108328503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,3,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,3,0.027535999814669292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,3,0.023120000958442688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,3,0.023706667125225067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,7,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,7,0.028757333755493164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,7,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,7,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,15,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,15,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,15,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,15,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,31,0.03350399931271871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,31,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,31,0.029738667110602062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,31,0.0296426663796107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,63,0.03325333446264267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,63,0.033439998825391136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,63,0.03121600051720937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,63,0.031141333281993866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,127,0.03358400116364161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,127,0.033674667278925575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,127,0.031258667508761086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,127,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,255,0.03383466601371765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,255,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,255,0.031370667119820915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,255,0.035455999275048576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,511,0.033573334415753685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,511,0.057189335425694786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,511,0.03145600110292435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,511,0.05390933156013489
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1023,0.03369600077470144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1023,0.09102400143941243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1023,0.031231999397277832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1023,0.08830933769543965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,2047,0.033615998923778534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,2047,0.1609119971593221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,2047,0.030970667799313862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,2047,0.1587999959786733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,4095,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,4095,0.3017759919166565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,4095,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,4095,0.2977973421414693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,8191,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,8191,0.5911466677983602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,8191,0.03136000037193298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,8191,0.5789759953816732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,3,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,3,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,3,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,7,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,7,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,7,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,7,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,15,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,15,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,15,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,15,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,127,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,255,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,255,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,255,0.011626667032639185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,511,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,511,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1023,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1023,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1023,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,2047,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,2047,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,2047,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,2047,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,4095,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,4095,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,4095,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,8191,0.014298666268587112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,8191,0.04224533339341482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,8191,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,8191,0.035962666074434914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,3,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,3,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,3,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,3,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,7,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,7,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,7,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,7,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,15,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,15,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,15,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,15,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,31,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,31,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,31,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,31,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,63,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,63,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,63,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,127,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,127,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,127,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,255,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,255,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,255,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,255,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,511,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,511,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,511,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,511,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1023,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1023,0.016858667135238647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1023,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,2047,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,2047,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,2047,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,2047,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,4095,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,4095,0.03941866755485535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,4095,0.015685333559910457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,4095,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,8191,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,8191,0.062224000692367554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,8191,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,8191,0.053914666175842285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,4095,0.05013866722583771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,3,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,3,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,3,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,7,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,7,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,7,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,7,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,15,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,15,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,15,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,15,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,31,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,31,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,31,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,63,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,31,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,63,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,63,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,127,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,127,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,127,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,127,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,255,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,255,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,255,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,511,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,511,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,511,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,511,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1023,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1023,0.023311999936898548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1023,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1023,0.021642667551835377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,2047,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,2047,0.03968533376852671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,2047,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,2047,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,4095,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,4095,0.062181333700815834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,4095,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,4095,0.054805333415667214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,8191,0.01770666614174843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,8191,0.10532800356547038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,8191,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1,0.009279999881982803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1,0.009269333134094873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,8191,0.09098133444786072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,3,0.009002666920423508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,3,0.009136000027259191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,7,0.009226666763424873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,7,0.009162666896979014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,15,0.009466666728258133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,15,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,31,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,31,0.009482666850090027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,63,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,255,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,255,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,255,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,255,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,511,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,511,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,511,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1023,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1023,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1023,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1023,0.027749332288901012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,2047,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,2047,0.031210665901501972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,2047,0.02370133250951767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,2047,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,4095,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,4095,0.04160533348719279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,4095,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,4095,0.043738668163617454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,8191,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,8191,0.06644799808661143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,8191,0.024127999941507976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,8191,0.06328000128269196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,16383,0.020096000283956528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,16383,0.11312533418337505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,16383,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,16383,0.10142933328946431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,32767,0.020831999679406483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,32767,0.20565332969029745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,32767,0.02499733368555705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,32767,0.17976532379786173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,65535,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,65535,0.3934719959894816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,65535,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,65535,0.3368053436279297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,3,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,7,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,127,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,255,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1023,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1023,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,2047,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,2047,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,2047,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,2047,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,4095,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,4095,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,4095,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,8191,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,8191,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,8191,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,8191,0.018464000274737675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,16383,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,16383,0.021130666136741638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,16383,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,16383,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,32767,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,32767,0.023077333966890972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,32767,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,32767,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,65535,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,65535,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,65535,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,65535,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,7,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,7,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,15,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,15,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,31,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,63,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,127,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,255,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,255,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,255,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,511,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,511,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1023,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1023,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,2047,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,2047,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,2047,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,4095,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,4095,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,4095,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,4095,0.01545599972208341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,8191,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,8191,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,8191,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,8191,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,16383,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,16383,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,16383,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,16383,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,32767,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,32767,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,32767,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,32767,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,65535,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,65535,0.039994666973749794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,65535,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,65535,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,3,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,15,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,63,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,63,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,511,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1023,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1023,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,2047,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,2047,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,2047,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,2047,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,4095,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,4095,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,4095,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,4095,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,8191,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,8191,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,8191,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,8191,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,16383,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,16383,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,16383,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,16383,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,32767,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,32767,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,32767,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,32767,0.031712000568707786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,65535,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,65535,0.06225066880385081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,65535,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,65535,0.052015999952952065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1,0.048250665267308555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1,0.04797333478927612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1,0.040362666050593056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,3,0.04775466521581014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,3,0.04772266745567322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,3,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,3,0.03957866628964742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,7,0.04806933303674062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,7,0.04951466619968414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,7,0.04146666576464971
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,7,0.04156800111134847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,15,0.04981866478919983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,15,0.05013866722583771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,15,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,15,0.0425546665986379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,31,0.059664001067479454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,31,0.06020266811052958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,31,0.05213333169619242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,31,0.052239999175071716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,63,0.060421332716941833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,63,0.0598880002895991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,63,0.0537013312180837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,63,0.054101333022117615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,127,0.06015466650327047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,127,0.06021333237489065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,127,0.053898667295773826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,127,0.054245332876841225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,255,0.059978668888409935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,255,0.07009066641330719
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,255,0.05409599840641022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,255,0.06440000236034393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,511,0.05996266504128774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,511,0.1034346620241801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,511,0.05375466744105021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,511,0.09716799855232239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1023,0.059978668888409935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1023,0.16901334126790366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1023,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1023,0.16293332974116007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,2047,0.060458665092786155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,2047,0.30390934149424237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,2047,0.05395199855168661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,2047,0.2966666618982951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,4095,0.06049066781997681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,4095,0.583184003829956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,4095,0.0539680023988088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,4095,0.5664693514506022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,3,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,3,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,3,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,3,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,7,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,7,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,7,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,7,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,15,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,15,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,15,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,15,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,31,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,31,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,31,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,31,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,63,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,63,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,63,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,63,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,127,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,127,0.0145066666106383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,127,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,127,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,255,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,255,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,255,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,255,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,511,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,511,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,511,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1023,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1023,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1023,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,2047,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,2047,0.02779199928045273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,2047,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,2047,0.025594666600227356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,4095,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,4095,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,4095,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,4095,0.03810133288304011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,3,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,3,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,3,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,3,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,7,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,7,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,7,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,15,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,15,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,15,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,15,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,31,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,31,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,31,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,31,0.014688000082969666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,63,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,63,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,63,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,63,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,127,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,127,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,127,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,127,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,255,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,255,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,255,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,255,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,511,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,511,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,511,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,511,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1023,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1023,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1023,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1023,0.023039999107519787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,2047,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,2047,0.04015466570854187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,2047,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,2047,0.03339733431736628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,4095,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,4095,0.06258133550484975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,4095,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,4095,0.05438933273156484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,3,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,3,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,3,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,3,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,7,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,7,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,7,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,7,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,15,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,15,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,15,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,15,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,31,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,31,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,31,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,31,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,63,0.018901333212852478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,63,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,63,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,63,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,127,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,127,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,127,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,127,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,255,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,255,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,255,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,255,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,511,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,511,0.02499199906984965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,511,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,511,0.02309333284695943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1023,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1023,0.03580799947182337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1023,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1023,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,2047,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,2047,0.058559998869895935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,2047,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,2047,0.05197333296140035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,4095,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,4095,0.09716266393661499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,4095,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,4095,0.08480532964070638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1,0.08916800220807393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1,0.08954133590062459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1,0.07438933352629344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1,0.07494933406511943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,3,0.09106133381525676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,3,0.09086400270462036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,3,0.07421333094437917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,3,0.0740479975938797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,7,0.09123200178146362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,7,0.09282666444778442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,7,0.07667199770609538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,7,0.07670400043328603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,15,0.09479467074076335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,15,0.09473066528638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,15,0.07887466748555501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,15,0.07893866797288258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,31,0.11340799927711487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,31,0.11331199606259663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,31,0.09889066219329834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,31,0.09912000099817912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,63,0.11334932843844096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,63,0.11342400312423706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,63,0.09938666224479675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,63,0.0993173321088155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,127,0.11396800478299458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,127,0.11369599898656209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,127,0.09934932986895244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,127,0.10121599833170573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,255,0.11428266763687134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,255,0.1339413324991862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,255,0.09955733021100362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,255,0.12001599868138631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,511,0.11545067032178243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,511,0.19931199153264365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,511,0.1002293328444163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,511,0.18643200397491455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1023,0.11549333731333415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1023,0.33406933148701984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1023,0.10127466917037964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1023,0.3182826638221741
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,3,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,3,0.01684800038735072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,3,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,3,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,7,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,7,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,7,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,7,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,15,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,15,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,15,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,15,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,31,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,31,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,31,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,31,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,63,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,63,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,63,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,63,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,127,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,127,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,127,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,127,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,255,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,255,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,255,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,255,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,511,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,511,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,511,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,511,0.019621333728233974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1023,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1023,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1023,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1023,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1,0.01865600049495697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1,0.018672000616788864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,3,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,3,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,3,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,3,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,7,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,7,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,7,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,7,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,15,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,15,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,15,0.0176959993938605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,15,0.01937599976857503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,31,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,31,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,31,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,31,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,63,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,63,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,63,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,63,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,127,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,127,0.01961600035429001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,127,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,127,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,255,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,255,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,255,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,255,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,511,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,511,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,511,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,511,0.02312533309062322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1023,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1023,0.03739733248949051
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1023,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1023,0.031445334355036415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,3,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,3,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,3,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,3,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,7,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,7,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,7,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,7,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,15,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,15,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,15,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,15,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,31,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,31,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,31,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,31,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,63,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,63,0.02718399961789449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,63,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,63,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,127,0.02697066714366277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,127,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,127,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,127,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,255,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,255,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,255,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,255,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,511,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,511,0.04048000027736028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,511,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,511,0.03513066718975703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1023,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1023,0.058362667759259544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1023,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1023,0.05402133365472158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,1,0.1714293360710144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,1,0.1727466583251953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,1,0.14010133345921835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,1,0.14019733667373657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,3,0.17296000321706137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,3,0.17282134294509888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,3,0.1400159994761149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,3,0.14019200205802917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,7,0.17690666516621908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,7,0.1771786610285441
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,7,0.14616533120473227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,7,0.14478933811187744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,15,0.18124266465504965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,15,0.18127999703089395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,15,0.15065600474675497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,15,0.15054399768511453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,31,0.21821333964665732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,31,0.2181653380393982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,31,0.1913493275642395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,31,0.19156799713770548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,63,0.22010666131973267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,63,0.21980265776316324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,63,0.19153600931167603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,63,0.19330666462580362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,127,0.22213866313298544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,127,0.22212799390157065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,127,0.19359999895095825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,127,0.19358932971954346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,255,0.2217866579691569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,255,0.26077866554260254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,255,0.19330666462580362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,255,0.23449599742889404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,511,0.22209600607554117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,511,0.39137065410614014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,511,0.19350934028625488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,511,0.3642079830169678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,1,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,1,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,1,0.020928000410397846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,1,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,3,0.02162666618824005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,3,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,3,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,3,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,7,0.02189333240191142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,7,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,7,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,7,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,15,0.021557333568731945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,15,0.022837333381175995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,15,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,15,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,31,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,31,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,31,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,31,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,63,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,63,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,63,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,63,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,127,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,127,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,127,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,127,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,255,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,255,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,255,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,255,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,511,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,511,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,511,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,511,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,1,0.027722666660944622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,1,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,1,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,1,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,3,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,3,0.02874133239189784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,3,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,3,0.026933332284291584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,7,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,7,0.027600000301996868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,7,0.027189334233601887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,7,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,15,0.02756800005833308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,15,0.027866666515668232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,15,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,15,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,31,0.027424000203609467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,31,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,31,0.02693866689999898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,31,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,63,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,63,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,63,0.025621332228183746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,63,0.0258240004380544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,127,0.02743999908367793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,127,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,127,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,127,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,255,0.029071999092896778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,255,0.029509333272775013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,255,0.027114666998386383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,255,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,511,0.029189333319664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,511,0.04029866556326548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,511,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,511,0.03576533248027166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,1,0.04211199780305227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,1,0.04358933369318644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,1,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,1,0.04048000027736028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,3,0.04381866753101349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,3,0.041696002086003624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,3,0.03963200002908707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,3,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,7,0.04383466641108195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,7,0.04345599810282389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,7,0.03984533250331879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,7,0.03969600051641464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,15,0.0436160018046697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,15,0.04341333111127218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,15,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,15,0.04005333284536997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,31,0.04381333291530609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,31,0.043791999419530235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,31,0.03978666663169861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,31,0.040762667854626976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,63,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,63,0.04385599990685781
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,63,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,63,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,127,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,127,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,127,0.039594667653242745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,127,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,255,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,255,0.04586133360862732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,255,0.04170133173465729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,255,0.041536000867684685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,511,0.04417066772778829
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,511,0.06577600042025249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,511,0.041690667470296226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,511,0.0610346645116806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1,0.014287999520699183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,3,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,3,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,7,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,7,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,7,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,15,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,15,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,31,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,31,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,63,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,63,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,63,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,63,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,127,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,127,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,127,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,255,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,255,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,255,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,255,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,511,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,511,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,511,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1023,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1023,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1023,0.020687999824682873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,2047,0.016789333273967106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,2047,0.03537066777547201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,2047,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,2047,0.030896000564098358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,4095,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,4095,0.055946667989095054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,4095,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,4095,0.051445335149765015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,8191,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,8191,0.09704533219337463
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,8191,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,8191,0.09090133508046468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,16383,0.015829333414634068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,16383,0.18098666270573935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,16383,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,16383,0.17281067371368408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,32767,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,32767,0.3476800123850505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,32767,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,32767,0.35118401050567627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,65535,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,65535,0.8271413644154867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,65535,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,65535,0.7520426909128824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,7,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,15,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,127,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,255,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,511,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,511,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,511,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1023,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1023,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1023,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1023,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,2047,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,2047,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,2047,0.013647999614477158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,4095,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,4095,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,4095,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,4095,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,8191,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,8191,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,8191,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,8191,0.018351999421914417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,16383,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,16383,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,16383,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,16383,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,32767,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,32767,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,32767,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,32767,0.027034667630990345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,65535,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,65535,0.04368533194065094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,65535,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,65535,0.03719999889532725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,3,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,3,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,7,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,15,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,31,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,31,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,127,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,127,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,255,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,511,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,511,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,511,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,511,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1023,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1023,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,2047,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,2047,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,2047,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,4095,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,4095,0.01676799977819125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,4095,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,8191,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,8191,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,8191,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,8191,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,16383,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,16383,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,16383,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,16383,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,32767,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,32767,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,32767,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,32767,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,65535,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,65535,0.06204266846179962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,65535,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,65535,0.05258666475613912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,3,0.011690666278203329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,7,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,7,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,7,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,15,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,511,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,511,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1023,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1023,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,2047,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,2047,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,2047,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,4095,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,4095,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,4095,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,4095,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,8191,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,8191,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,8191,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,8191,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,16383,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,16383,0.0391839991013209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,16383,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,16383,0.031210665901501972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,32767,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,32767,0.062319998939832054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,32767,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,32767,0.05271466573079427
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,65535,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,65535,0.10587732990582784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,65535,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,65535,0.08892266949017842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,1,0.3360053300857544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,1,0.33688533306121826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,1,0.27349867423375446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,1,0.2733440001805623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,3,0.3365600109100342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,3,0.3368320067723592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,3,0.2733013431231181
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,3,0.27373866240183514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,7,0.3470933437347412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,7,0.34705066680908203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,7,0.2828373312950134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,7,0.28335465987523395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,15,0.3539573351542155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,15,0.35493866602579754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,15,0.29600000381469727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,15,0.29578133424123126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,31,0.4310133457183838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,31,0.42980798085530597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,31,0.37563733259836835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,31,0.3759146531422933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,63,0.43281598885854083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,63,0.4329440196355184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,63,0.3777546485265096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,63,0.37778135140736896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,127,0.43680532773335773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,127,0.4368213415145874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,127,0.37996800740559894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,127,0.3793066740036011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,255,0.43702399730682373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,255,0.5155093272527059
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,255,0.3796159823735555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,255,0.45842134952545166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,1,0.03339199970165888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,1,0.033674667278925575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,1,0.03294399877389272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,1,0.03338133295377096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,3,0.033733333150545754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,3,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,3,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,3,0.03321066747109095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,7,0.033530667424201965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,7,0.03333866596221924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,7,0.03145066648721695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,7,0.03253333270549774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,15,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,15,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,15,0.033301333586374916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,15,0.03313066562016805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,31,0.03365333378314972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,31,0.03510399907827377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,31,0.031397332747777305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,31,0.03149333347876867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,63,0.03363200028737386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,63,0.03329599897066752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,63,0.03173333406448364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,63,0.03173866619666418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,127,0.03342399994532267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,127,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,127,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,127,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,255,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,255,0.0337119996547699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,255,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,255,0.0331839993596077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,1,0.04594666759173075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,1,0.045594667394955955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,1,0.04362666606903076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,1,0.0440586656332016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,3,0.04560000201066335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,3,0.045882667104403176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,3,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,3,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,7,0.045647998650868736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,7,0.04576000074545542
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,7,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,7,0.04368533194065094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,15,0.04560000201066335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,15,0.0459199994802475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,15,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,15,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,31,0.04601066807905833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,31,0.045781334241231285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,31,0.044069334864616394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,31,0.04374399781227112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,63,0.04584000011285146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,63,0.04621866842110952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,63,0.04378133515516917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,63,0.043621331453323364
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,127,0.046944002310434975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,127,0.045882667104403176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,127,0.04362666606903076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,127,0.04365866879622141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,255,0.04756266872088114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,255,0.0480373352766037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,255,0.04568533102671305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,255,0.04364266494909922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,1,0.07755733529726665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,1,0.0769760012626648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,1,0.0705866664648056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,1,0.07055999835332234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,3,0.07645333309968312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,3,0.07737066845099132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,3,0.07110400001207988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,3,0.07069866855939229
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,7,0.07752533257007599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,7,0.0768746683994929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,7,0.0724533349275589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,7,0.07261333366235097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,15,0.078575998544693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,15,0.07705600063006084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,15,0.07052266597747803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,15,0.07065066695213318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,31,0.07674666742483775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,31,0.0777759999036789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,31,0.072543998559316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,31,0.07194666564464569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,63,0.07673066854476929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,63,0.07740800082683563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,63,0.07271466652552287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,63,0.07257066667079926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,127,0.07871999839941661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,127,0.07859200239181519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,127,0.07212799787521362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,127,0.07203733424345653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,255,0.08089066545168559
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,255,0.07924800117810567
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,255,0.07293866574764252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,255,0.07468800246715546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,1,0.6642239888509115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,1,0.6665120124816895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,1,0.5386186838150024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,1,0.5378453334172567
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,3,0.6650133530298868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,3,0.6645813385645548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,3,0.5388160149256388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,3,0.539253314336141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,7,0.6860746542612711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,7,0.6866453488667806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,7,0.5581760009129842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,7,0.560421347618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,15,0.7021439870198568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,15,0.7009706497192383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,15,0.5828160047531128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,15,0.5828799804051717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,31,0.8538666566212972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,31,0.8533973693847656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,31,0.7445440292358398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,31,0.7443040211995443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,63,0.8577866554260254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,63,0.8580426375071207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,63,0.7479893366495768
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,63,0.7485706806182861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,127,0.8711146513621012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,127,0.8730186621348063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,127,0.7518239816029867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,127,0.7531733512878418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,1,0.05779733260472616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,1,0.05789866546789805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,1,0.05418666700522105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,1,0.054085334142049156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,3,0.05585066477457682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,3,0.05618133147557577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,3,0.0539626677831014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,3,0.05421333511670431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,7,0.056234667698542275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,7,0.05681600173314413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,7,0.054048001766204834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,7,0.054154664278030396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,15,0.0561706672112147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,15,0.05787733197212219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,15,0.05399466554323832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,15,0.05403199791908264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,31,0.05632533133029938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,31,0.0565226674079895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,31,0.05413866539796194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,31,0.054330666859944664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,63,0.055957332253456116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,63,0.05620799958705902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,63,0.05384000142415365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,63,0.05399466554323832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,127,0.058330665032068886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,127,0.05821866790453593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,127,0.0543093333641688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,127,0.05426133175690969
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,1,0.0827466646830241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,1,0.08329600095748901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,1,0.07850666840871175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,1,0.07682666679223378
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,3,0.0827946662902832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,3,0.08249600231647491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,3,0.0767146646976471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,3,0.07838400204976399
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,7,0.08292800188064575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,7,0.08267199993133545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,7,0.0787306676308314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,7,0.07863999903202057
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,15,0.08287466565767924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,15,0.0828959991534551
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,15,0.07894399762153625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,15,0.07689600189526875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,31,0.08286400139331818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,31,0.08285866677761078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,31,0.07698133091131847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,31,0.07769600053628285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,63,0.08267733454704285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,63,0.08247999846935272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,63,0.07663466533025105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,63,0.07693333427111308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,127,0.08268799881140391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,127,0.08269866804281871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,127,0.07674666742483775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,127,0.07879999776681264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,1,0.14416000247001648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,1,0.14664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,1,0.1328373352686564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,1,0.13432533542315164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,3,0.14446399609247842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,3,0.1457919975121816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,3,0.13395733634630838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,3,0.1352000037829081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,7,0.144378662109375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,7,0.14547200004259744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,7,0.1340053379535675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,7,0.1339946687221527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,15,0.14444266756375632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,15,0.14451733231544495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,15,0.13422933220863342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,15,0.135754664738973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,31,0.1444533367951711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,31,0.13405866424242655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,31,0.14408533771832785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,31,0.13396799564361572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,63,0.14446933070818582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,63,0.1443946659564972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,63,0.13395733634630838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,63,0.13362133502960205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,127,0.14441066980361938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,127,0.14429866274197897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,127,0.13576533397038779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1,0.014229333649079004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,127,0.1339359978834788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,3,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,3,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,3,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,3,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,7,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,7,0.013482666263977686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,7,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,7,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,15,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,15,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,15,0.015872000406185787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,15,0.016154666741689045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,31,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,31,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,31,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,31,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,63,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,63,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,63,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,63,0.020981334149837494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,127,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,127,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,127,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,127,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,255,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,255,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,255,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,255,0.020746666938066483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,511,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,511,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,511,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,511,0.024106666445732117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1023,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1023,0.03328000009059906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1023,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1023,0.033930666744709015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,2047,0.05394133428732554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,2047,0.02146666745344798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,2047,0.05412800113360087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,4095,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,4095,0.0956160028775533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,4095,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,4095,0.0932426651318868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,8191,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,8191,0.18118933836619058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,8191,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,8191,0.17243733008702597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,16383,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,16383,0.3571893374125163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,16383,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,16383,0.33062400420506793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,32767,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,32767,1.0624533494313557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,32767,0.020874666670958202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,32767,0.690720001856486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,7,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,15,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,63,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,63,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,511,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,511,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1023,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1023,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,2047,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,2047,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,2047,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,4095,0.013514666507641474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,4095,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,4095,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,4095,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,8191,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,8191,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,8191,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,8191,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,16383,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,16383,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,16383,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,16383,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,32767,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,32767,0.04399466514587402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,32767,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,32767,0.03676266719897588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,3,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,7,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,15,0.01221866657336553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,63,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,127,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,511,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,511,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,511,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1023,0.012634667257467905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1023,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,2047,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,2047,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,2047,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,4095,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,4095,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,4095,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,4095,0.01836799954374631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,8191,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,8191,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,8191,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,8191,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,16383,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,16383,0.04156800111134847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,16383,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,16383,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,32767,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,32767,0.06237866481145223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,32767,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,32767,0.053823997577031456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,3,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,7,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,63,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,15,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,255,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,255,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,511,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,511,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1023,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1023,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,2047,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,2047,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,2047,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,2047,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,4095,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,4095,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,4095,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,4095,0.024853333830833435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,8191,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,8191,0.038005332152048744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,8191,0.013754667093356451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,8191,0.03164266546567281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,16383,0.06022400160630544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,16383,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,16383,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,16383,0.05220800141493479
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,32767,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,32767,0.10352533062299092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,32767,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,32767,0.09082667032877605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,3,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,3,0.016149333367745083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,7,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,7,0.016693333784739178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,15,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,15,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,31,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,31,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,63,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,127,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,127,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,255,0.03155199935038885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,255,0.03130666663249334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,511,0.04967466493447622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,511,0.04805333415667216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1023,0.0849120020866394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1023,0.08425600330034892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,2047,0.15812800327936807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,2047,0.15457066893577576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,4095,0.2998826702435811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,4095,0.2955999970436096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,8191,0.5974560181299845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,8191,0.5787893136342367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,16383,1.4225279490152996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,16383,1.4897173245747883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,3,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,31,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,63,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,511,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,511,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1023,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1023,0.014015999933083853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,2047,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,2047,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,4095,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,4095,0.02120000123977661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,8191,0.04107200105985006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,8191,0.0276853342851003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,16383,0.06155733267466227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,16383,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,15,0.012367999802033106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,31,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,31,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,63,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,63,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,127,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,127,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,255,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,255,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,511,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1023,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1023,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,2047,0.02162133405605952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,2047,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,4095,0.039408000806967415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,4095,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,8191,0.05900266766548157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,8191,0.039706667264302574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,16383,0.10054933031400044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,16383,0.05813866853713989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,3,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,3,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,7,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,7,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,15,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,15,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,31,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,31,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,63,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,63,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,127,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,127,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,255,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,255,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,511,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,511,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1023,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1023,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,2047,0.038319999972979225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,2047,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,4095,0.06005333364009857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,4095,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,8191,0.10121066371599834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,8191,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,16383,0.18540799617767334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,16383,0.10203733046849568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1,0.009119999905427298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,3,0.009216000015536943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,3,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,7,0.00922133338948091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,15,0.009248000259200731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,31,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,63,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,63,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,255,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,255,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,511,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,511,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1023,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1023,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,2047,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,2047,0.029359998802344005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,4095,0.037685332198937736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,4095,0.03930133332808813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,8191,0.056159997979799904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,8191,0.05608533322811127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,16383,0.09313066800435384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,16383,0.08498666683832805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,32767,0.16729066769282022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,32767,0.1460906664530436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,65535,0.316591997941335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,65535,0.27127466599146527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,7,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,7,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,15,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,63,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1023,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,2047,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,4095,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,4095,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,8191,0.01757866640885671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,8191,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,16383,0.02146666745344798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,16383,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,32767,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,32767,0.023344000180562336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,65535,0.027082666754722595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,65535,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,3,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,15,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,31,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,127,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,2047,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,2047,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,4095,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,4095,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,8191,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,8191,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,16383,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,16383,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,32767,0.022672000030676525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,32767,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,65535,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,65535,0.023029332359631855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,63,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,63,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,255,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1023,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1023,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,2047,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,4095,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,4095,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,8191,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,8191,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,16383,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,16383,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,32767,0.021930667261282604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,32767,0.019461333751678467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,65535,0.03757333258787791
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,65535,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1,0.009061333412925402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,3,0.008853333070874214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,7,0.008997333546479544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,15,0.00892800030608972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,31,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,63,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,63,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,127,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,255,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,255,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,511,0.022895999252796173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,511,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1023,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1023,0.029909332593282063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,2047,0.03404266635576884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,2047,0.03743999948104223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,4095,0.04996799925963084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,4095,0.05042133231957754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,8191,0.08455999692281087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,8191,0.07714133461316426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,16383,0.1444533367951711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,16383,0.12800533572832742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,32767,0.2671999931335449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,32767,0.23011734088261923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,65535,0.5137600104014078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,65535,0.4357173442840576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,3,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,2047,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,2047,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,4095,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,4095,0.014357333381970724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,8191,0.019215999792019527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,8191,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,16383,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,16383,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,32767,0.023530667026837666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,32767,0.021877333521842957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,65535,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,65535,0.025018667181332905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,3,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,7,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,31,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,63,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,127,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,511,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1023,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1023,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,2047,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,2047,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,4095,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,4095,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,8191,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,8191,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,16383,0.021002667645613354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,16383,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,32767,0.02332799881696701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,32767,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,65535,0.0377866675456365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,65535,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,511,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,511,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1023,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,2047,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,2047,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,4095,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,4095,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,8191,0.018778666853904724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,8191,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,16383,0.02294933299223582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,16383,0.020762667059898376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,32767,0.03832533210515976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,32767,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,65535,0.057146668434143066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,65535,0.03756266583998998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,3,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,3,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,7,0.029525332152843475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,7,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,15,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,15,0.03088533381621043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,31,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,31,0.031541332602500916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,63,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,63,0.031557333966096245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,127,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,127,0.03579200059175491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,255,0.05592533449331919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,255,0.05372266471385956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,511,0.09087466200192769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,511,0.08779733379681905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1023,0.16080533464749655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1023,0.15451199809710184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,2047,0.2998453378677368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,2047,0.291647990544637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,4095,0.5849440097808838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,4095,0.5614826679229736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,8191,1.1699946721394856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,8191,1.116159995396932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,3,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,7,0.012624000509579977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,7,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,15,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,15,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,63,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,63,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,127,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,127,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,255,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,255,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,511,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1023,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1023,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,2047,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,2047,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,4095,0.03992533435424169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,4095,0.02759466568628947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,8191,0.0601440022389094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,8191,0.04182933270931244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,3,0.013637332866589228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,3,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,7,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,7,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,15,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,15,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,31,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,31,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,63,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,63,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,127,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,127,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,255,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,255,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,511,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,511,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1023,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1023,0.01674666628241539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,2047,0.03887466589609782
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,2047,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,4095,0.06027733286221822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,4095,0.040031999349594116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,8191,0.1013813316822052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,8191,0.061834668119748436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1,0.01664000004529953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,3,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,3,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,7,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,7,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,15,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,15,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,31,0.016224000602960587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,31,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,63,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,63,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,127,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,127,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,255,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,255,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,511,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,511,0.01836799954374631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1023,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1023,0.02363733450571696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,2047,0.06374399860699971
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,2047,0.041893333196640015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,4095,0.10260267059008281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,4095,0.06208533545335134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,8191,0.1860533356666565
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,8191,0.10358400146166484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1,0.009082666908701261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,3,0.009242666885256767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,7,0.009349333122372627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,15,0.009066666786869368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,63,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,255,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,255,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,511,0.025381334125995636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,511,0.02757866680622101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1023,0.03158933420976003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1023,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,2047,0.04342933495839437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,2047,0.042352000872294106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,4095,0.06657066444555919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,4095,0.06223999957243601
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,8191,0.1135093371073405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,8191,0.10097600022951762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,16383,0.2062293291091919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,16383,0.1789919932683309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,32767,0.3920533259709676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,32767,0.3333653211593628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,65535,0.7854613463083903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,65535,0.6426399946212769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,3,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,7,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,31,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,511,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1023,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1023,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,2047,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,2047,0.014778666198253632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,4095,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,4095,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,8191,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,8191,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,16383,0.022111999491850536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,16383,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,32767,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,32767,0.023344000180562336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,65535,0.04223466912905375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,65535,0.02951466788848241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,63,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,127,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,511,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1023,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,2047,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,2047,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,4095,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,4095,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,8191,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,8191,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,16383,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,16383,0.02254933367172877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,32767,0.04125866790612539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,32767,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,65535,0.05962666869163513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,65535,0.03979199876387914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,3,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,7,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,511,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1023,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,2047,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,2047,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,4095,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,8191,0.021669333179791767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,8191,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,16383,0.03875199953715006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,16383,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,32767,0.05973333120346069
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,32767,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,65535,0.10116266210873921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,65535,0.06072533130645752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1,0.04799466828505198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1,0.041797334949175514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,3,0.04828266799449921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,3,0.04196266829967499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,7,0.050106664498647056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,7,0.04379733403523763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,15,0.058559998869895935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,15,0.054085334142049156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,31,0.05840000013510386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,31,0.05406400064627329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,63,0.058517331878344216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,63,0.05376000205675761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,127,0.06866666674613953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,127,0.06422933439413707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,255,0.10314666231473286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,255,0.09711999694506328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,511,0.16897066434224448
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,511,0.16110400358835855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1023,0.303877333799998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1023,0.2913493315378825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,2047,0.5781333446502686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,2047,0.5485653479894003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,4095,1.1240959962209065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,4095,1.0761439800262451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,3,0.013493333011865616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,3,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,7,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,7,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,15,0.013797332843144735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,15,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,31,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,31,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,63,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,63,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,127,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,127,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,255,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,511,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,511,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1023,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1023,0.018837332725524902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,2047,0.04050666590531667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,2047,0.028357334434986115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,4095,0.06225066880385081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,4095,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,3,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,3,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,7,0.016810666769742966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,15,0.015706667055686314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,15,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,31,0.016490666816631954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,31,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,63,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,63,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,127,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,127,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,255,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,255,0.015664000064134598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,511,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,511,0.019498666127522785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1023,0.035946667194366455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1023,0.023503998915354412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,2047,0.06177600224812826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,2047,0.04162133236726125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,4095,0.10282666484514873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,4095,0.062496001521746315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,3,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,3,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,7,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,7,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,15,0.021290667355060577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,15,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,31,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,31,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,63,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,63,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,127,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,127,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,255,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,255,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,511,0.03352533280849457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,511,0.023584000766277313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1023,0.053914666175842285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1023,0.035642666121323906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,2047,0.09598400195439656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,2047,0.0598880002895991
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,4095,0.1730133295059204
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,4095,0.09544000029563904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1,0.08821333448092143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1,0.07471466561158498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,3,0.09114666779836018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,3,0.07712533573309581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,7,0.09293333689371745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,7,0.08072533210118611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,15,0.11150933305422465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,15,0.09939733147621155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,31,0.11174399654070537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,31,0.10071466366449992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,63,0.11176000038782756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,63,0.1011893351872762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,127,0.13205333550771078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,127,0.12177600463231404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,255,0.19751467307408652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,255,0.1856106718381246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,511,0.3285440007845561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,511,0.31406400601069134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1023,0.601034681002299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1023,0.5667999982833862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,3,0.017530667285124462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,3,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,7,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,7,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,15,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,15,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,31,0.018650667121013004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,31,0.017242666333913803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,63,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,63,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,127,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,127,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,255,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,255,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,511,0.023178666830062866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,511,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1023,0.03749866783618927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1023,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,3,0.021712000171343487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,3,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,7,0.02143999934196472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,7,0.01942933350801468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,15,0.021727999051411945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,15,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,31,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,31,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,63,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,63,0.01950399950146675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,127,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,127,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,255,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,255,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,511,0.03502399971087774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,511,0.025413334369659424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1023,0.05421866476535797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1023,0.036506667733192444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1,0.03091199944416682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,3,0.029472000896930695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,3,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,7,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,7,0.027434666951497395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,15,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,15,0.02810666710138321
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,31,0.03152533372243246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,31,0.027786667148272198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,63,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,63,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,127,0.02977599948644638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,127,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,255,0.03399466723203659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,255,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,511,0.054976001381874084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,511,0.03965333352486292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1023,0.09286933143933614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1023,0.058101331194241844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,1,0.16874132553736368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,1,0.14193600416183472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,3,0.17515732844670615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,3,0.1462613344192505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,7,0.17735467354456583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,7,0.1541920006275177
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,15,0.21588265895843506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,15,0.19146132469177246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,31,0.21589332818984985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,31,0.19337066014607748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,63,0.21624000867207846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,63,0.19349332650502524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,127,0.2567039926846822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,127,0.23430933554967245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,255,0.38648533821105957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,255,0.363045334815979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,511,0.6627519925435384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,511,0.6174506743748983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,1,0.025455998877684276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,1,0.0235359991590182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,3,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,3,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,7,0.025146665672461193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,7,0.023370665808518726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,15,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,15,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,31,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,31,0.02351466566324234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,63,0.024847999215126038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,63,0.023354666928450268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,127,0.025237334271272022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,127,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,255,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,255,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,511,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,511,0.029546665648619335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,1,0.03336533407370249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,1,0.029722665747006733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,3,0.03311999887228012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,3,0.029653333127498627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,7,0.03329066683848699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,7,0.02956266701221466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,15,0.033488000432650246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,15,0.029818666477998097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,31,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,31,0.029557332396507263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,63,0.031658666829268135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,63,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,127,0.03153600047032038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,127,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,255,0.037392000357309975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,255,0.029520000020662945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,511,0.05815466741720835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,511,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,1,0.050373335679372154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,1,0.043840001026789345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,3,0.04984533290068308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,3,0.0440586656332016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,7,0.05005866785844167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,7,0.04387199878692627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,15,0.05009600023428599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,15,0.04394133388996124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,31,0.04996799925963084
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,31,0.04390933116277059
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,63,0.05013333261013031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,63,0.04418666660785675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,127,0.05276800195376078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,127,0.044069334864616394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,255,0.0563679983218511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,255,0.04621866842110952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,511,0.09594133496284485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,511,0.06629866858323415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,3,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,7,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,15,0.014655999839305878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,15,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,31,0.016730666160583496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,31,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,63,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,63,0.01268799975514412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,127,0.016714667280515034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,255,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,255,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,511,0.024277334411938984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,511,0.021754667162895203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1023,0.03436266630887985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1023,0.03181866556406021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,2047,0.05638933181762695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,2047,0.05221333106358846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,4095,0.09780266880989075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,4095,0.09131200114885966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,8191,0.1808639963467916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,8191,0.17093332608540854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,16383,0.3455359935760498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,16383,0.33054933945337933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,32767,0.6861813068389893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,32767,0.7397653261820475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,65535,1.7526613871256511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,65535,1.5958986282348633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,7,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,31,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,31,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,511,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1023,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1023,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,2047,0.016885332763195038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,2047,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,4095,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,4095,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,8191,0.0207893339296182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,8191,0.01945066700379054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,16383,0.025061334172884624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,16383,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,32767,0.041759997606277466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,32767,0.02942933390537898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,65535,0.06132799883683523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,65535,0.04380266865094503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,7,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1023,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1023,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,2047,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,2047,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,4095,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,4095,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,8191,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,8191,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,16383,0.03957333415746689
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,16383,0.025098666548728943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,32767,0.05917333563168844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,32767,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,65535,0.10013866424560547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,65535,0.060191998879114784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,3,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,63,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,255,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,511,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,511,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1023,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1023,0.014069333672523499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,2047,0.017738666385412216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,2047,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,4095,0.021557333568731945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,4095,0.019386666516462963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,8191,0.04008000095685323
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,8191,0.025573333104451496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,16383,0.059861332178115845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,16383,0.040037333965301514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,32767,0.10155199964841206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,32767,0.06195199986298879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,65535,0.18516266345977783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,65535,0.10132267077763875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,1,0.3287786642710368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,1,0.2754986683527629
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,3,0.34273068110148114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,3,0.28570665915807086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,7,0.34913067022959393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,7,0.3002026677131653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,15,0.42313599586486816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,15,0.37677868207295734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,31,0.4246240059534709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,31,0.3777173360188802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,63,0.42505065600077313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,63,0.37772266070048016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,127,0.5032266775767008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,127,0.4596266746520996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,255,0.7837493419647217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,255,0.7158506711324056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,1,0.04001066585381826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,1,0.037717332442601524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,3,0.03963200002908707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,3,0.039162665605545044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,7,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,7,0.03738133360942205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,15,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,15,0.037903999288876854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,31,0.039594667653242745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,31,0.03789866715669632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,63,0.03965866565704346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,63,0.037834666669368744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,127,0.039781334499518074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,127,0.03808533400297165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,255,0.043706665436426796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,255,0.03990933299064636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,1,0.05641066531340281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,1,0.050250664353370667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,3,0.055957332253456116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,3,0.05007466673851013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,7,0.05514133473237356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,7,0.049642667174339294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,15,0.054469332098960876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,15,0.05017599960168203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,31,0.05635199944178263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,31,0.05019199848175049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,63,0.05555733541647593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,63,0.04997866849104563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,127,0.05589333176612854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,127,0.050106664498647056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,255,0.062368000547091164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,255,0.052186667919158936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,1,0.09111467003822327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,1,0.07884266475836436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,3,0.09122133255004883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,3,0.07939733564853668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,7,0.09134933352470398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,7,0.07913599908351898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,15,0.09101333220799764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,15,0.07881066699822743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,31,0.09161067008972168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,31,0.07919466495513916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,63,0.09130133191744487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,63,0.07881600161393483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,127,0.09171199798583984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,127,0.0807360013326009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,255,0.10155199964841206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,255,0.08281599978605907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,1,0.6503200133641561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,1,0.5419360001881918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,3,0.6795466740926107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,3,0.5638240178426107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,7,0.6932586828867594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,7,0.5929973522822062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,15,0.8402400016784668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,15,0.744874636332194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,31,0.8404853343963623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,31,0.7476320266723633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,63,0.851744016011556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,63,0.7491040229797363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,127,1.0401333173116047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,127,0.9245440165201823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,1,0.06850666801134746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,1,0.06239999830722809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,3,0.0681386689345042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,3,0.0628053347269694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,7,0.06878399848937988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,7,0.06238933404286703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,15,0.06764799853165944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,15,0.06243733565012614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,31,0.06849599877993266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,31,0.06263466676076253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,63,0.06643199920654297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,63,0.06277333199977875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,127,0.06846933563550313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,127,0.06446399788061778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,1,0.09816533327102661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,1,0.08898666501045227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,3,0.09731200337409973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,3,0.08915733297665913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,7,0.09759466846783955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,7,0.08890133102734883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,15,0.09710933764775594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,15,0.08852799733479817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,31,0.09737066427866618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,31,0.08736532926559448
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,63,0.09701333443323772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,63,0.08903466661771138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,127,0.09817066788673401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,127,0.08931199709574382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,1,0.16699733336766562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,1,0.1467573344707489
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,3,0.1669600009918213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,3,0.14738133549690247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,7,0.16698133945465088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,7,0.14654399951299033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,15,0.16689600547154745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,15,0.1464959979057312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,31,0.16716265678405762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,31,0.1465013325214386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,63,0.16725865999857584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,63,0.1465226709842682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,127,0.16686934232711792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,127,0.14826666315396628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,3,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,3,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,7,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,7,0.015461333096027374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,15,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,15,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,31,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,31,0.020224000016848247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,63,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,63,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,127,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,127,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,255,0.021375998854637146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,255,0.023557332654794056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,511,0.0316746657093366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,511,0.03358400116364161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1023,0.05398400127887726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1023,0.05429866909980774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,2047,0.0960586667060852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,2047,0.09338133533795674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,4095,0.18092799186706543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,4095,0.17088532447814941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,8191,0.3492000102996826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,8191,0.3267306685447693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,16383,0.7096693515777588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,16383,0.6364426612854004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,32767,2.483978589375814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,32767,1.631813367207845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,511,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,511,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1023,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1023,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,2047,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,2047,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,4095,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,4095,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,8191,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,8191,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,16383,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,16383,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,32767,0.060533334811528526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,32767,0.04348800083001455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,15,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,15,0.01228800043463707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,31,0.012389333297808966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,31,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,63,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,255,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,511,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1023,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1023,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,2047,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,2047,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,4095,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,4095,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,8191,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,8191,0.025962665677070618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,16383,0.060175999999046326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,16383,0.040149333576361336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,32767,0.10139200091362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,32767,0.06076266864935557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,63,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,511,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,511,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1023,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1023,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,2047,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,2047,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,4095,0.03860799968242645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,4095,0.0242399995525678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,8191,0.05885866781075796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,8191,0.038506666819254555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,16383,0.1011840005715688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,16383,0.05900266766548157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,32767,0.1837706764539083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,32767,0.09804266691207886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,7,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,31,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,63,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,255,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,255,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,255,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,511,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,511,0.0124746672809124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1023,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1023,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1023,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,2047,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,2047,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,2047,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,4095,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,4095,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,4095,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,4095,0.017551999539136887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,8191,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,8191,0.0216799999276797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,8191,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,8191,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,16383,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,16383,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,16383,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,16383,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,32767,0.014736000448465347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,32767,0.044778664906819664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,32767,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,32767,0.035760000348091125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,3,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,3,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,7,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,7,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,127,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,511,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,511,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,511,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,511,0.011962667107582092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1023,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1023,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1023,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,2047,0.014053333550691605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,2047,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,2047,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,4095,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,4095,0.019530666371186573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,4095,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,4095,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,8191,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,8191,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,8191,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,8191,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,16383,0.013845333208640417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,16383,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,16383,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,16383,0.03154666721820831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,32767,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,32767,0.06275733311971028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,32767,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,32767,0.05407999952634176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,3,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,3,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,3,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,7,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,7,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,7,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,31,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,63,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,127,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,127,0.012554666648308435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,255,0.011674666156371435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,255,0.01209066684047381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,255,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,255,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,511,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,511,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,511,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,511,0.011695999652147293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1023,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1023,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1023,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,2047,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,2047,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,2047,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,2047,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,4095,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,4095,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,4095,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,4095,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,8191,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,8191,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,8191,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,8191,0.03152533372243246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,16383,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,16383,0.061493332187334694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,16383,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,16383,0.054234668612480164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,32767,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,32767,0.10366933544476827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,32767,0.014090667168299357
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,32767,0.08902933200200398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,3,0.012645332763592402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,3,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,3,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,3,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,7,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,7,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,7,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,7,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,15,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,15,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,15,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,31,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,31,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,31,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,31,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,63,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,63,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,63,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,127,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,127,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,255,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,255,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,255,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,255,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,511,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,511,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,511,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,511,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1023,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1023,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1023,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,2047,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,2047,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,2047,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,2047,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,4095,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,4095,0.04038933416207632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,4095,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,4095,0.033333333830038704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,8191,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,8191,0.06239999830722809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,8191,0.015530666957298914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,8191,0.05578133463859558
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,16383,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,16383,0.10525866349538167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,16383,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,16383,0.09086933732032776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,32767,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,32767,0.18942934274673462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,32767,0.015610666324694952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,32767,0.16285333037376404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,3,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,3,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,3,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,7,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,7,0.011920000116030375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,7,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,7,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,15,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,15,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,15,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,31,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,31,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,63,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,63,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,63,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,255,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,511,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,511,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1023,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1023,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,2047,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,2047,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,2047,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,2047,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,4095,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,4095,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,4095,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,4095,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,8191,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,8191,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,8191,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,8191,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,16383,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,16383,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,16383,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,16383,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,32767,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,32767,0.02735999971628189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,32767,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,32767,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,65535,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,65535,0.031189332405726116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,65535,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,65535,0.031210665901501972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,131071,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,131071,0.03501333296298981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,131071,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,131071,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,3,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,7,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,7,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,63,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,63,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,127,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,127,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,511,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,511,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1023,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1023,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1023,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,2047,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,2047,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,2047,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,2047,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,4095,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,4095,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,4095,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,4095,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,8191,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,8191,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,8191,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,8191,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,16383,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,16383,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,16383,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,16383,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,32767,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,32767,0.021536000072956085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,32767,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,32767,0.021402666966120403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,65535,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,65535,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,65535,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,65535,0.023237332701683044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,131071,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,131071,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,131071,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,7,0.010464000205198923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,15,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,31,0.010426666587591171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,63,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,127,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,255,0.010485333700974783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,511,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1023,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1023,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,2047,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,2047,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,2047,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,2047,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,4095,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,4095,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,4095,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,4095,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,8191,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,8191,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,8191,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,8191,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,16383,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,16383,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,16383,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,16383,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,32767,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,32767,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,32767,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,32767,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,65535,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,65535,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,65535,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,65535,0.020954666038354237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,2047,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,131071,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,131071,0.02992533395687739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,131071,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,131071,0.027600000301996868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,3,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,3,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,7,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,7,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,7,0.010453333457310995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,31,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,31,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,63,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,127,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,511,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,511,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1023,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1023,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,2047,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,2047,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,2047,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,2047,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,4095,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,4095,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,4095,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,4095,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,8191,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,8191,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,8191,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,8191,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,16383,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,16383,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,16383,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,16383,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,32767,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,32767,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,32767,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,32767,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,65535,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,65535,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,65535,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,65535,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,131071,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,131071,0.039674667020638786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,131071,0.013381333400805792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,131071,0.033759998778502144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,3,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,3,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,7,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,15,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,63,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,127,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,127,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,511,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,511,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,511,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1023,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1023,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1023,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,2047,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,2047,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,2047,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,2047,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,4095,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,4095,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,4095,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,4095,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,8191,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,8191,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,8191,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,8191,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,16383,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,16383,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,16383,0.020992000897725422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,32767,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,32767,0.0230880007147789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,32767,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,32767,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,65535,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,65535,0.023669332265853882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,65535,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,65535,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,131071,0.013658666362365087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,131071,0.029493334392706554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,131071,0.013376000026861826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,131071,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,3,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,7,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,15,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,31,0.010458666831254959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,127,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,511,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1023,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1023,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1023,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,2047,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,2047,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,2047,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,2047,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,4095,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,4095,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,4095,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,8191,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,63,0.020981334149837494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,8191,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,8191,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,8191,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,16383,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,16383,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,16383,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,16383,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,32767,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,32767,0.02082666630546252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,32767,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,32767,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,65535,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,65535,0.02293866624434789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,65535,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,65535,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,131071,0.013445333888133367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,131071,0.029557332396507263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,131071,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,131071,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,3,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,15,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,15,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,15,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,63,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,255,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,511,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1023,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1023,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,2047,0.012416000167528788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,2047,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,2047,0.011488000551859537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,2047,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,4095,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,4095,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,4095,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,4095,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,8191,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,8191,0.01623999948302905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,8191,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,8191,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,16383,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,16383,0.017407999684413273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,16383,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,16383,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,32767,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,32767,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,32767,0.012309333930412928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,32767,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,65535,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,65535,0.025205334027608235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,65535,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,65535,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,131071,0.013530666629473368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,131071,0.04109866668780645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,131071,0.013568000247081121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,131071,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,63,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,63,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,127,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,127,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,255,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,511,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1023,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1023,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,2047,0.012448000411192576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,2047,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,2047,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,2047,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,4095,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,4095,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,4095,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,4095,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,8191,0.012298667182525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,8191,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,8191,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,8191,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,16383,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,16383,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,16383,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,16383,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,32767,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,32767,0.025648000339667004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,32767,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,32767,0.02342933416366577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,65535,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,65535,0.0395413339138031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,65535,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,65535,0.03202133377393087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,131071,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,131071,0.0603359987338384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,131071,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,131071,0.05205333232879639
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,7,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,7,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,7,0.011978667229413986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,15,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,31,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,63,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,63,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,511,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,511,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,511,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1023,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1023,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,2047,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,2047,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,2047,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,2047,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,4095,0.013365333278973898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,4095,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,4095,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,4095,0.019968000551064808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,8191,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,8191,0.02787200113137563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,8191,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,8191,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,16383,0.014682666709025701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,16383,0.04477333525816599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,16383,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,16383,0.037621334195137024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,3,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,7,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,15,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,15,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,63,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,255,0.012015999605258306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,511,0.012063999970753988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1023,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1023,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1023,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,2047,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,2047,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,2047,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,2047,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,4095,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,4095,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,4095,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,4095,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,8191,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,8191,0.03942933430274328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,8191,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,8191,0.03226666649182638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,16383,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,16383,0.06041066845258077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,16383,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,16383,0.05363733569780985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,3,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,3,0.012560000022252401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,3,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,3,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,7,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,7,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,7,0.012560000022252401
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,15,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,15,0.012085333466529846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,15,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,15,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,31,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,31,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,31,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,31,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,63,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,63,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,63,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,127,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,127,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,127,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,255,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,255,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,255,0.01257066677014033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,511,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,511,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,511,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1023,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1023,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1023,0.016741332908471424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,2047,0.015850666910409927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,2047,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,2047,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,2047,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,4095,0.01618133361140887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,4095,0.03984000037113825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,4095,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,4095,0.03326933334271113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,8191,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,8191,0.06249066690603892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,8191,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,8191,0.054010664423306785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,16383,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,16383,0.10565333565076192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,16383,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,16383,0.09144000212351482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,3,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,3,0.01463466634353002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,3,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,3,0.015386667102575302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,7,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,7,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,7,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,7,0.013594667116800943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,15,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,15,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,15,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,15,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,31,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,31,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,31,0.014602666099866232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,31,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,63,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,63,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,63,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,63,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,127,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,127,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,127,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,127,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,255,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,255,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,255,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,255,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,511,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,511,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,511,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,511,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1023,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1023,0.023354666928450268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1023,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1023,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,2047,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,2047,0.03996799886226654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,2047,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,2047,0.033226666351159416
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,4095,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,4095,0.06188266475995382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,4095,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,4095,0.055311997731526695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,8191,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,8191,0.10534399747848511
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,8191,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,8191,0.09141332904497783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,16383,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,16383,0.19037866592407227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,16383,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,16383,0.16285333037376404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,3,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,3,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,7,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,31,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,127,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,127,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,255,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,511,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,511,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1023,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1023,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,2047,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,2047,0.013621332744757334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,2047,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,4095,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,4095,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,4095,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,4095,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,8191,0.013306666165590286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,8191,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,8191,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,8191,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,16383,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,16383,0.020090666910012562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,16383,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,16383,0.019962667177120846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,32767,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,32767,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,32767,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,32767,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,65535,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,65535,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,65535,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,65535,0.02480533222357432
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,131071,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,131071,0.03374933451414108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,131071,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,131071,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,63,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,127,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,127,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,255,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,255,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,255,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,511,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,511,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,511,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1023,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1023,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1023,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,2047,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,2047,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,2047,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,4095,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,4095,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,4095,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,4095,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,8191,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,8191,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,8191,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,8191,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,16383,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,16383,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,16383,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,16383,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,32767,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,32767,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,32767,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,32767,0.01988799994190534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,65535,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,65535,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,65535,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,65535,0.024160000185171764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,131071,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,131071,0.042538667718569435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,131071,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,131071,0.03379199902216593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,3,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,3,0.011760000139474869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,7,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,15,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,31,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,31,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,63,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,63,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,127,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,511,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1023,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1023,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1023,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,2047,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,2047,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,4095,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,4095,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,4095,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,4095,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,8191,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,8191,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,8191,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,8191,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,16383,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,16383,0.020928000410397846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,16383,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,16383,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,32767,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,32767,0.025642665723959606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,32767,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,32767,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,65535,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,65535,0.039408000806967415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,65535,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,65535,0.03165333221356074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,131071,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,131071,0.06206933160622915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,131071,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,131071,0.05373333394527435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,31,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,127,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,127,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,511,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,511,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,511,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1023,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1023,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1023,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1023,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,2047,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,2047,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,2047,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,2047,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,4095,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,4095,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,4095,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,4095,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,8191,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,8191,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,8191,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,8191,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,16383,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,16383,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,16383,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,16383,0.02143466720978419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,32767,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,32767,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,32767,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,32767,0.03146133323510488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,65535,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,65535,0.062234664956728615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,65535,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,65535,0.052469333012898765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,131071,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,131071,0.10897599657376607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,131071,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,131071,0.09170132875442505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,3,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,15,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,15,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,15,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,31,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,127,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,127,0.012506666282812754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,511,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,511,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1023,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1023,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1023,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,2047,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,2047,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,2047,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,2047,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,4095,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,4095,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,4095,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,4095,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,8191,0.013877333452304205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,8191,0.04242666562398275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,8191,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,8191,0.03541333228349686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,3,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,3,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,3,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,3,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,7,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,7,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,7,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,7,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,15,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,15,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,15,0.013482666263977686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,31,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,31,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,31,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,31,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,63,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,63,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,63,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,63,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,127,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,127,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,127,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,127,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,255,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,255,0.012442667037248611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,255,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,511,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,511,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,511,0.014581333845853806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1023,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1023,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,2047,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,2047,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,2047,0.015919999529918034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,2047,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,4095,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,4095,0.03977599988381068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,4095,0.015669333438078564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,4095,0.033413333197434746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,8191,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,8191,0.06200533111890157
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,8191,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,8191,0.05444799860318502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,3,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,3,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,3,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,3,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,7,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,7,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,7,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,7,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,15,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,15,0.014767999450365702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,15,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,15,0.014725333700577417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,31,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,31,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,31,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,31,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,63,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,63,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,63,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,63,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,127,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,127,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,127,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,127,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,255,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,255,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,255,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,255,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,511,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,511,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,511,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,511,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1023,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1023,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1023,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1023,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,2047,0.016821333517630894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,2047,0.04010133445262909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,2047,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,2047,0.03332799921433131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,4095,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,4095,0.06284800171852112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,4095,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,4095,0.05598933498064677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,8191,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,8191,0.10513066252072652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,8191,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,8191,0.09123733639717102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,3,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,3,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,3,0.01758933315674464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,3,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,7,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,7,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,7,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,7,0.01788266624013583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,15,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,15,0.017653333644072216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,15,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,15,0.017685333887736004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,31,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,31,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,31,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,31,0.0180479995906353
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,63,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,63,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,63,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,63,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,127,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,127,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,127,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,127,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,255,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,255,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,255,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,255,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,511,0.018874666343132656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,511,0.023077333966890972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,511,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,511,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1023,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1023,0.035461333890755974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1023,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1023,0.02995733420054118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,2047,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,2047,0.05879466732343038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,2047,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,2047,0.05249600112438202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,4095,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,4095,0.09724266330401103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,4095,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,4095,0.08494399984677632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,8191,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,8191,0.17085866133371988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,8191,0.02102400114138921
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,8191,0.14802666505177817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,3,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,3,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,3,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,3,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,7,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,7,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,7,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,7,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,15,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,15,0.014597332725922266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,15,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,15,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,31,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,31,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,31,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,31,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,63,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,63,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,63,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,63,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,127,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,127,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,127,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,255,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,255,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,255,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,511,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,511,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,511,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,511,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1023,0.018373332917690277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1023,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1023,0.016965333372354507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,2047,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,2047,0.02934933453798294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,2047,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,2047,0.026213333010673523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,3,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,3,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,3,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,7,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,7,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,7,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,7,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,15,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,15,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,15,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,15,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,31,0.015290666371583939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,31,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,31,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,31,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,63,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,63,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,63,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,63,0.014720000326633453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,127,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,127,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,127,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,127,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,255,0.014709333578745524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,255,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,255,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,511,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,511,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,511,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,511,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1023,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1023,0.024869332710901897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1023,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1023,0.02277333289384842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,2047,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,2047,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,2047,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,2047,0.03355200091997782
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,3,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,3,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,3,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,3,0.018789333601792652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,7,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,7,0.018768000106016796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,7,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,7,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,15,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,15,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,15,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,15,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,31,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,31,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,31,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,31,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,63,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,63,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,63,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,63,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,127,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,127,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,127,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,127,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,255,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,255,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,255,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,255,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,511,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,511,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,511,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,511,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1023,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1023,0.03583466758330663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1023,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1023,0.031370667119820915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,2047,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,2047,0.06020799775918325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,2047,0.021226666867733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,2047,0.05277333160241445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1,0.025663999219735462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,3,0.025637333591779072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,3,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,3,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,3,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,7,0.026127999027570088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,7,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,7,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,7,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,15,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,15,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,15,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,15,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,31,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,31,0.027679999669392902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,31,0.025616000096003216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,31,0.02513066679239273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,63,0.02699733277161916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,63,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,63,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,63,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,127,0.02640533447265625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,127,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,127,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,127,0.025754667818546295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,255,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,255,0.027503999571005504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,255,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,255,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,511,0.027376001079877216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,511,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,511,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,511,0.03527999917666117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1023,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1023,0.05797333518664042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1023,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1023,0.05193600058555603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,2047,0.029178666571776073
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,2047,0.09920000036557515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,2047,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,2047,0.08858666817347209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,3,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,3,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,3,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,3,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,7,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,7,0.01569066693385442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,7,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,7,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,15,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,15,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,15,0.016384000579516094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,15,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,31,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,31,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,31,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,31,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,63,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,63,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,63,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,63,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,127,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,127,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,127,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,127,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,255,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,255,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,255,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,255,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,511,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,511,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,511,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,511,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1023,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1023,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1023,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1023,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,3,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,3,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,3,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,3,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,7,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,7,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,7,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,7,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,15,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,15,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,15,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,15,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,31,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,31,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,31,0.017637333522240322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,31,0.01728533332546552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,63,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,63,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,63,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,63,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,127,0.020101333657900494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,127,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,127,0.02033599962790807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,127,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,255,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,255,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,255,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,255,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,511,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,511,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,511,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,511,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1023,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1023,0.03578133384386698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1023,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1023,0.03129599988460541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1,0.025850666066010792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,3,0.027722666660944622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,3,0.027589333554108936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,3,0.025685332715511322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,3,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,7,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,7,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,7,0.025557334224383037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,7,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,15,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,15,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,15,0.02514133354028066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,15,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,31,0.027114666998386383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,31,0.027029333015282948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,31,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,31,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,63,0.027114666998386383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,63,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,63,0.02510933329661687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,63,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,127,0.027263998985290527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,127,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,127,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,127,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,255,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,255,0.027280000348885853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,255,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,255,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,511,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,511,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,511,0.026554666459560394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,511,0.03527999917666117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1023,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1023,0.05817066629727682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1023,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1023,0.05190933247407278
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1,0.04182933270931244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1,0.042453333735466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1,0.039642666776975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1,0.03951466580231985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,3,0.042261332273483276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,3,0.04171733558177948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,3,0.039450667798519135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,3,0.03963200002908707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,7,0.04192000130812327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,7,0.041984001795450844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,7,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,7,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,15,0.04188799858093262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,15,0.04171200096607208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,15,0.03959999978542328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,15,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,31,0.04196799794832865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,31,0.04195199906826019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,31,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,31,0.03997866561015447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,63,0.04185600082079569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,63,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,63,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,63,0.0395413339138031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,127,0.041989331444104515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,127,0.04159466673930486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,127,0.03941866755485535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,127,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,255,0.04365866879622141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,255,0.044064000248909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,255,0.041589332123597465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,255,0.039546666045983635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,511,0.04353066782156626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,511,0.06429333488146464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,511,0.03979199876387914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,511,0.05813866853713989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1023,0.04390400151411692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1023,0.10124267141024272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1023,0.04162133236726125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1023,0.09062932928403218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,7,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,15,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,31,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,31,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,127,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,127,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,127,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,255,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,255,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1023,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1023,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,2047,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,2047,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,2047,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,4095,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,4095,0.020031999796628952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,4095,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,4095,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,8191,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,8191,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,8191,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,8191,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,16383,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,16383,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,16383,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,16383,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,32767,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,32767,0.02306666721900304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,32767,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,32767,0.021722666919231415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,65535,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,65535,0.027322667340437572
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,65535,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,65535,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,131071,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,131071,0.045935998360315956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,131071,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,131071,0.03762666632731756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,7,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,7,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,63,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,127,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,511,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1023,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1023,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1023,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,2047,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,2047,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,2047,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,2047,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,4095,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,4095,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,4095,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,4095,0.016864000509182613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,8191,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,8191,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,8191,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,8191,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,16383,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,16383,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,16383,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,32767,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,32767,0.025263999899228413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,32767,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,32767,0.023472001155217487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,65535,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,65535,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,65535,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,65535,0.03156800071398417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,131071,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,131071,0.06333866715431213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,131071,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,131071,0.05412266651789347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,3,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,63,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,127,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,255,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,511,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,511,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1023,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1023,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1023,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,2047,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,2047,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,2047,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,4095,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,4095,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,4095,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,4095,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,8191,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,8191,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,8191,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,8191,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,16383,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,16383,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,16383,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,16383,0.023237332701683044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,32767,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,32767,0.03950933367013931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,32767,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,32767,0.031162666777769726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,65535,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,65535,0.06204266846179962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,65535,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,65535,0.054042667150497437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,131071,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,131071,0.10791466633478801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,131071,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,131071,0.09126933415730794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,3,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,15,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,31,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,63,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,127,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,255,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,255,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,255,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,511,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,511,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1023,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1023,0.01322666679819425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1023,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,2047,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,2047,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,2047,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,2047,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,4095,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,4095,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,4095,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,4095,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,8191,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,8191,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,8191,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,8191,0.02311466634273529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,16383,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,16383,0.03957333415746689
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,16383,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,16383,0.03200000027815501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,32767,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,32767,0.06235733131567637
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,32767,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,32767,0.05399466554323832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,65535,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,65535,0.10514133175214131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,65535,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,65535,0.0890826682249705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,131071,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,131071,0.19688532749811807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,131071,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,131071,0.16219199697176614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,1,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,1,0.021551998953024547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,1,0.021125334004561108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,1,0.020928000410397846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,3,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,3,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,3,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,3,0.021429332594076794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,7,0.021055998901526134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,7,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,7,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,7,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,15,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,15,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,15,0.021104000508785248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,15,0.020960000654061634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,31,0.021541332205136616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,31,0.021221332252025604
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,31,0.021162666380405426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,31,0.02143999934196472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,63,0.021295999487241108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,63,0.021594665944576263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,63,0.020741333564122517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,63,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,127,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,127,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,127,0.021381333470344543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,127,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,255,0.02110933264096578
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,255,0.021205333371957142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,255,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,255,0.021168000996112823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,511,0.021349333226680756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,511,0.029215998947620392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,511,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,511,0.02718399961789449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,1,0.027552001178264618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,1,0.027797333896160126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,1,0.02735466758410136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,1,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,3,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,3,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,3,0.025578667720158894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,3,0.025775998830795288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,7,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,7,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,7,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,7,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,15,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,15,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,15,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,15,0.02775999903678894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,31,0.027664000789324444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,31,0.027744000156720478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,31,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,31,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,63,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,63,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,63,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,63,0.02568000058333079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,127,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,127,0.027642667293548584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,127,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,127,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,255,0.02917333443959554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,255,0.02773866554101308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,255,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,255,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,511,0.027653334041436512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,511,0.04045866678158442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,511,0.027162666122118633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,511,0.0353973334034284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,1,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,1,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,1,0.039781334499518074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,1,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,3,0.04358399907747904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,3,0.04188799858093262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,3,0.03982399900754293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,3,0.03950933367013931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,7,0.04177066683769226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,7,0.04192000130812327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,7,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,7,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,15,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,15,0.04372799893220266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,15,0.039834665755430855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,15,0.039749334255854286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,31,0.04354133208592733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,31,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,31,0.039850667119026184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,31,0.04001066585381826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,63,0.04349866509437561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,63,0.04354666670163473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,63,0.03972266614437103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,63,0.03950933367013931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,127,0.041850666205088295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,127,0.0435146689414978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,127,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,127,0.039701332648595176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,255,0.04454400142033895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,255,0.04528533418973287
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,255,0.04165333261092504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,255,0.04114133367935816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,511,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,131071,0.025861332813898723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,511,0.06632000207901001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,511,0.04199466605981191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,511,0.06011733412742615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,1,0.07446399827798207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,1,0.07441066702206929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,1,0.06810133159160614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,1,0.06888000170389812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,3,0.07441066702206929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,3,0.06845866640408833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,3,0.06830400228500366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,7,0.07444266478220622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,7,0.0744053324063619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,7,0.06820266445477803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,7,0.06834133466084798
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,15,0.0743999977906545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,3,0.07541333138942719
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,15,0.07446399827798207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,15,0.06915199756622314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,15,0.06834666430950165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,31,0.07483733197053273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,31,0.07442133128643036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,31,0.06844800213972728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,31,0.06854933500289917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,63,0.07449066638946533
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,63,0.07445333401362102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,63,0.068271999557813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,63,0.06843199829260509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,127,0.07635200023651123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,127,0.0761599987745285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,127,0.06854933500289917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,127,0.06823466718196869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,255,0.07682133217652638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,255,0.07620266576608022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,255,0.07046400010585785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,255,0.07034666836261749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,511,0.07702933251857758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,511,0.1163093348344167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,511,0.07168533404668172
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,511,0.10492266217867534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,1,0.033471999069054924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,1,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,1,0.03161066770553589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,1,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,3,0.033717334270477295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,3,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,255,0.0120319997270902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,3,0.03141866624355316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,3,0.03387733300526937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,7,0.03364799916744232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,7,0.03382399926582972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,7,0.03179733455181122
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,7,0.03342933456103007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,15,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,15,0.033743999898433685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,15,0.03178133318821589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,15,0.03230933348337809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,31,0.033546666304270424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,31,0.033359999457995095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,31,0.03178666780392329
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,31,0.03183466692765554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,63,0.033333333830038704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,63,0.03183999905983607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,63,0.03395200024048487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,63,0.03148799886306127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,127,0.03345066557327906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,127,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,127,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,127,0.0317493329445521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,255,0.0335413341720899
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,255,0.033439998825391136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,255,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,1,0.046165332198143005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,1,0.04611733555793762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,1,0.04409599800904592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,1,0.043738668163617454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,3,0.04632000128428141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,3,0.04584000011285146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,255,0.03401600072781245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,3,0.042026668787002563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,3,0.04201066493988037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,7,0.04560533165931702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,7,0.04589866598447164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,7,0.041840001940727234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,7,0.04234133164087931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,15,0.04545066754023234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,15,0.045834665497144066
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,15,0.042624001701672874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,15,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,31,0.04574933151404063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,31,0.045909335215886436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,31,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,31,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,63,0.04566933214664459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,63,0.04586666822433472
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,63,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,63,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,127,0.045935998360315956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,127,0.04603200157483419
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,127,0.043824002146720886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,127,0.04282666742801666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,255,0.04769066472848257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,255,0.047930667797724404
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,255,0.04390933116277059
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,255,0.044165333112080894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,1,0.07646400233109792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,1,0.07073066631952922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,1,0.0707893321911494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,3,0.07678399980068207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,3,0.07672533392906189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,3,0.07047466437021892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,3,0.0705813318490982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,7,0.07653333246707916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,1,0.0793333351612091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,7,0.07067733506361644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,7,0.07273600002129872
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,15,0.07646400233109792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,15,0.07832000156243642
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,15,0.07249600191911061
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,15,0.07258666555086772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,31,0.07654933134714763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,31,0.07670933504899342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,31,0.07045866549015045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,31,0.07225066423416138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,63,0.07683200140794118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,63,0.07088533540566762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,63,0.07085866729418437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,127,0.07869866490364075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,127,0.07866133252779643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,63,0.07696533203125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,127,0.07047466437021892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,127,0.07054933408896129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,255,0.08179200192292531
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,255,0.07870933413505554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,255,0.07246933380762736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,255,0.07275733351707458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,1,0.14046399792035422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,1,0.14035200079282126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,1,0.12825066844622293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,1,0.12771200140317282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,3,0.14037866393725076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,3,0.14005333185195923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,3,0.12797333796819052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,3,0.12779733538627625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,7,0.14040533701578775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,7,0.1402400036652883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,7,0.12784000237782797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,7,0.12802666425704956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,15,0.1404853363831838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,15,0.1280639966328939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,15,0.1404159963130951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,15,0.12780800461769104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,31,0.14014933506647745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,31,0.14019733667373657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,31,0.1281599998474121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,31,0.12770133217175803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,63,0.14038399855295816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,63,0.14028799533843994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,63,0.12777066230773926
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,63,0.12809066971143088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,127,0.1413706640402476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,127,0.14009599884351095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,127,0.13014933466911316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,127,0.13004799683888754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,255,0.14441600441932678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,255,0.14072533448537192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,255,0.1341813306013743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,3,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,255,0.12970133622487387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,127,0.010464000205198923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,255,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,255,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,511,0.01258133351802826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1023,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1023,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,2047,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,2047,0.016735999534527462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,2047,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,2047,0.016293333222468693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,4095,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,4095,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,4095,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,8191,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,8191,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,8191,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,8191,0.01933866615096728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,16383,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,16383,0.02479466547568639
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,16383,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,16383,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,32767,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,32767,0.0284853329261144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,32767,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,32767,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,65535,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,65535,0.0455626646677653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,65535,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,65535,0.03621866554021835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,3,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,15,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,63,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,127,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,127,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,511,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1023,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,2047,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,2047,0.014495999862750372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,2047,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,4095,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,4095,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,4095,0.0138026662170887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,4095,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,8191,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,8191,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,8191,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,8191,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,16383,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,16383,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,16383,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,16383,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,32767,0.013669333110253016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,32767,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,32767,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,32767,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,65535,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,65535,0.06229333579540253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,65535,0.01349866638580958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,65535,0.05389333268006643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,7,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,15,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,255,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,511,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1023,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1023,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1023,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,2047,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,2047,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,2047,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,2047,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,4095,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,4095,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,4095,0.01379199946920077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,4095,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,8191,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,8191,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,8191,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,8191,0.02332266668478648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,16383,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,16383,0.041008000572522484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,16383,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,16383,0.031930667658646904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,32767,0.013722666849692663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,32767,0.06266666452089946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,32767,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,32767,0.054469332098960876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,65535,0.01333333303531011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,65535,0.1074773371219635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1,0.011823999385039011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1,0.011999999483426413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,65535,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,3,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,65535,0.08895466725031535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,127,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,511,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1023,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1023,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1023,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,2047,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,2047,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,2047,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,2047,0.018453333526849747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,4095,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,4095,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,4095,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,4095,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,8191,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,8191,0.03956266740957896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,8191,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,8191,0.031658666829268135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,16383,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,16383,0.060453335444132485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,16383,0.01607999950647354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,16383,0.05301866432030996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,32767,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,32767,0.10332799951235454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,32767,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,32767,0.08921600381533305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,65535,0.013471999516089758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,65535,0.18752533197402954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,65535,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,3,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,15,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,65535,0.16153066356976828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,63,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,511,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,511,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,2047,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,2047,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,4095,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,4095,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,8191,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,8191,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,16383,0.04044266790151596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,16383,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,32767,0.062234664956728615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,32767,0.04401599864164988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,7,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,63,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,511,0.01156266654531161
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1023,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1023,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,2047,0.019541333119074505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,2047,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,4095,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,4095,0.021557333568731945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,8191,0.03969600051641464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,8191,0.025429333249727886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,16383,0.039813332259655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,32767,0.10198400417963664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,32767,0.06202666461467743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,15,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,63,0.012266666938861212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1023,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1023,0.013338666409254074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,2047,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,2047,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,4095,0.038762666285037994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,4095,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,8191,0.058543999989827476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,8191,0.03885866701602936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,16383,0.09956799944241841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,16383,0.058042665322621666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,32767,0.1816800038019816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,32767,0.09925867120424907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,3,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,3,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,7,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,15,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,15,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,31,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,31,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,63,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,63,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,127,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,255,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,511,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1023,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1023,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,2047,0.037791999677817024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,2047,0.02502399931351344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,4095,0.060032000144322716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,4095,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,8191,0.10201600193977356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,8191,0.06076799829800924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,16383,0.18584533532460532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,16383,0.10095999638239543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,32767,0.3540480136871338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,32767,0.1847040057182312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,3,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,7,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,7,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,15,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,15,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,31,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,63,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,63,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,511,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1023,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,2047,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,2047,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,4095,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,4095,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,8191,0.017114666601022083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,8191,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,16383,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,16383,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,32767,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,32767,0.027530667682488758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,65535,0.03323733309904734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,65535,0.031957333286603294
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,131071,0.0366239994764328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,131071,0.03591466695070267
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,7,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,63,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,127,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,255,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1023,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1023,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,2047,0.011776000261306763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,4095,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,4095,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,8191,0.01693333312869072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,8191,0.016890666137139004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,16383,0.0206133338312308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,16383,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,32767,0.023152001202106476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,32767,0.023168000082174938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,65535,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,65535,0.023503998915354412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,131071,0.030693332354227703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,131071,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,7,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,15,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,31,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,127,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,255,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,511,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,511,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1023,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1023,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,2047,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,2047,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,4095,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,4095,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,8191,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,8191,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,16383,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,16383,0.019039999693632126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,32767,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,32767,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,65535,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,65535,0.023258666197458904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,131071,0.043354665239652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,131071,0.029343999922275543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,3,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,3,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,7,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,7,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,63,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,511,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,2047,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,4095,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,4095,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,8191,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,8191,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,16383,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,16383,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,32767,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,32767,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,65535,0.03682133307059606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,65535,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,131071,0.060032000144322716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,131071,0.03974399964014689
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,15,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,127,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,255,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1023,0.012522666404644648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1023,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,2047,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,4095,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,4095,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,8191,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,8191,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,16383,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,16383,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,32767,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,32767,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,65535,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,65535,0.027162666122118633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,131071,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,131071,0.03081600119670232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,3,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,31,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,63,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,511,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1023,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1023,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,2047,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,2047,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,4095,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,8191,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,8191,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,16383,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,16383,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,32767,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,32767,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,65535,0.02497066557407379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,65535,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,131071,0.04354133208592733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,131071,0.029781334102153778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,511,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1023,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1023,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,2047,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,2047,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,4095,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,4095,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,8191,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,8191,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,16383,0.020831999679406483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,16383,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,32767,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,32767,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,65535,0.03775466730197271
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,65535,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,131071,0.059450666109720864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,131071,0.04066666712363561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,3,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,15,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,31,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,127,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1023,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1023,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,2047,0.014783999572197596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,2047,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,4095,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,4095,0.016538667182127636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,8191,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,8191,0.016677333662907284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,16383,0.02310933421055476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,16383,0.02006400004029274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,32767,0.03743999948104223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,32767,0.025040000677108765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,65535,0.05819733440876007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,65535,0.03810133288304011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,131071,0.09839999675750732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,131071,0.05805333455403646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1023,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1023,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,2047,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,2047,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,4095,0.02465066562096278
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,4095,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,8191,0.038880000511805214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,8191,0.029461334149042766
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,16383,0.06192000210285187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,16383,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,3,0.011338666081428528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,7,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,7,0.011594666788975397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,15,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,31,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,31,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,127,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,255,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,511,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1023,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1023,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,2047,0.021503999829292297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,2047,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,4095,0.03835200021664301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,4095,0.02513599892457326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,8191,0.059749335050582886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,8191,0.03955733279387156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,16383,0.10244266192118327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,16383,0.0599839985370636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,3,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,3,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,7,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,7,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,15,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,15,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,31,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,31,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,63,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,127,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,255,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,511,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,511,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1023,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1023,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,2047,0.038133333126703896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,2047,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,4095,0.060378665725390114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,4095,0.040405333042144775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,8191,0.10193066795667012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,8191,0.06170133252938589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,16383,0.18423465887705484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,16383,0.10290666421254475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,3,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,3,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,7,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,7,0.014677333335081736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,15,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,15,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,31,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,31,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,63,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,63,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,127,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,127,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,255,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,255,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,511,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,511,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1023,0.034714666505654655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1023,0.02327466756105423
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,2047,0.06001066664854685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,2047,0.041365332901477814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,4095,0.10222400228182475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,4095,0.06201066573460897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,8191,0.18648000558217367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,8191,0.10321066776911418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,16383,0.35446401437123615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,16383,0.18600533405939737
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,7,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,511,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,511,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1023,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1023,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,2047,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,2047,0.015466666469971338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,4095,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,4095,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,8191,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,8191,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,16383,0.022106667359670002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,16383,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,32767,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,32767,0.023056000471115112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,65535,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,65535,0.02585600068171819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,131071,0.04683733483155569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,131071,0.03408000121514002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,7,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,127,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,511,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1023,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,2047,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,2047,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,4095,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,4095,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,8191,0.019461333751678467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,8191,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,16383,0.022687998910744984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,16383,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,32767,0.023103999594847362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,32767,0.021269333859284718
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,65535,0.03973866750796636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,65535,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,131071,0.062368000547091164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,131071,0.042122667034467064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,511,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1023,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,2047,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,2047,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,4095,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,4095,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,8191,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,8191,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,16383,0.02515733242034912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,16383,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,32767,0.039962666730086006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,32767,0.027509334186712902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,65535,0.06020799775918325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,65535,0.03982399900754293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,131071,0.10129066308339436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,131071,0.06092800199985504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,511,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1023,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1023,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,2047,0.016437333077192307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,2047,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,4095,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,4095,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,8191,0.02146133283774058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,8191,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,16383,0.03772266705830892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,16383,0.025087999800841015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,32767,0.05880533158779144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,32767,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,65535,0.10115733742713928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,65535,0.0602400004863739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,131071,0.1876266598701477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,131071,0.105621337890625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,7,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,15,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,31,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,31,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,63,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,63,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,127,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,127,0.012554666648308435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,255,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,511,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1023,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1023,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,2047,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,2047,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,4095,0.04008533308903376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,4095,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,8191,0.061050668358802795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,8191,0.042037333051363625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,3,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,3,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,7,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,7,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,15,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,15,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,31,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,31,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,63,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,63,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,127,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,127,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,255,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,255,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,511,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,511,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1023,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1023,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,2047,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,2047,0.025413334369659424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,4095,0.06076799829800924
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,4095,0.0395359992980957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,8191,0.10106666882832845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,8191,0.0621066689491272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,3,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,3,0.015360000232855478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,7,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,7,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,15,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,15,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,31,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,31,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,63,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,63,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,127,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,127,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,255,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,511,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,511,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1023,0.03375466664632162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1023,0.023242667317390442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,2047,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,2047,0.04040000090996424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,4095,0.10104533036549886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,4095,0.062208001812299095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,8191,0.1845866640408834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,8191,0.1035146713256836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1,0.021151999632517498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,3,0.019909333437681198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,3,0.01907733331123988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,7,0.021125334004561108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,7,0.019760000209013622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,15,0.021674667795499165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,15,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,31,0.019482667247454327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,31,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,63,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,63,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,127,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,127,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,255,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,255,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,511,0.03368533402681351
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,511,0.023520000278949738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1023,0.0528053343296051
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1023,0.0354666660229365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,2047,0.09571733077367146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,2047,0.05806399881839752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,4095,0.17100799083709717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,4095,0.09516266981760661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,8191,0.3267786701520284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,8191,0.16909333070119223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,3,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,3,0.014805333067973455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,7,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,7,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,15,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,15,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,31,0.013295999417702356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,31,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,63,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,63,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,127,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,127,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,255,0.014485333114862442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,511,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,511,0.01562133307258288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1023,0.02165866643190384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1023,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,2047,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,2047,0.02945599953333537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,3,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,7,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,7,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,15,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,15,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,31,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,31,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,63,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,63,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,127,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,127,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,255,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,255,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,511,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,511,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1023,0.035530666510264076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1023,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,2047,0.06206933160622915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,2047,0.04163199911514918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,3,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,3,0.01834133391578992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,7,0.01932799940307935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,7,0.018725333114465077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,15,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,15,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,31,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,31,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,63,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,63,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,127,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,127,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,255,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,255,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,511,0.03344533344109853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,511,0.02384000023206075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1023,0.053823997577031456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1023,0.035386666655540466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,2047,0.09717333316802979
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,2047,0.05939733485380808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,3,0.02938666691382726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,3,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,7,0.029205332199732464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,7,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,15,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,15,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,31,0.029482667644818623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,31,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,63,0.029311999678611755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,63,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,127,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,127,0.027114666998386383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,255,0.034074666599432625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,255,0.027717334528764088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,511,0.05440000196297964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,511,0.038693333665529885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1023,0.09200533231099446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1023,0.05609600245952606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,2047,0.17189866304397583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,2047,0.09732266267140706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,3,0.017632000148296356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,3,0.01704000060757001
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,7,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,7,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,15,0.017136000096797943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,15,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,31,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,31,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,63,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,63,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,127,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,127,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,255,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,255,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,511,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,511,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1023,0.03761066744724909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1023,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,3,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,3,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,7,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,7,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,15,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,15,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,31,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,31,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,63,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,63,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,127,0.021125334004561108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,127,0.019466667125622433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,255,0.02088533341884613
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,255,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,511,0.0340639998515447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,511,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1023,0.0553706685702006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1023,0.03557866563399633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1,0.029658667743206024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,3,0.029861333469549816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,3,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,7,0.029669334491093952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,7,0.027493332823117573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,15,0.029616000751654308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,15,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,31,0.0295413335164388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,31,0.026848000784715016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,63,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,63,0.02703999976317088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,127,0.029285334050655365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,127,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,255,0.03421866645415624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,255,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,511,0.05522133409976959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,511,0.03959999978542328
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1023,0.09373866518338521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1023,0.05766933163007101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1,0.04804266492525736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1,0.041519999504089355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,3,0.0476746658484141
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,3,0.04186133543650309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,7,0.047685335079828896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,7,0.041663999358812966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,15,0.04804266492525736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,15,0.04166933397452036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,31,0.04775466521581014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,31,0.04179200033346812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,63,0.04782933493455251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,63,0.04171200096607208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,127,0.04878933231035868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,127,0.04167466859022776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,255,0.05392000079154968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,255,0.04370133578777313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,511,0.09286399682362874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,511,0.06318399806817372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1023,0.16884799798329672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1023,0.0993173321088155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,63,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,127,0.011770666887362799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,127,0.011701333026091257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,255,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,511,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,511,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1023,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,2047,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,2047,0.015381333728631338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,4095,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,4095,0.017535999417304993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,8191,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,8191,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,16383,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,16383,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,32767,0.02510933329661687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,32767,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,65535,0.04162133236726125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,65535,0.029781334102153778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,131071,0.06448000172773997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,131071,0.04633066554864248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,7,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,15,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,127,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1023,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,2047,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,2047,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,4095,0.017312000195185345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,4095,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,8191,0.020314666132132213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,8191,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,16383,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,16383,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,32767,0.039674667020638786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,32767,0.025253333151340485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,65535,0.059418668349583946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,65535,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,131071,0.09990400075912476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,131071,0.06187733511130015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1023,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,2047,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,2047,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,4095,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,4095,0.016895999511082966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,8191,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,8191,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,16383,0.038831998904546104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,16383,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,32767,0.05978666742642721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,32767,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,65535,0.10038933157920837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,65535,0.06029333174228668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,131071,0.18549867471059164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,131071,0.10366400082906087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,3,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,31,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,31,0.014453332871198654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,63,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,255,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,511,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,511,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1023,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1023,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,2047,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,2047,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,4095,0.021514666577180225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,4095,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,8191,0.038133333126703896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,8191,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,16383,0.05960000058015188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,16383,0.039034667114416756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,32767,0.1011840005715688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,32767,0.06027733286221822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,65535,0.1837600072224935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,65535,0.10172266761461894
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,131071,0.3544906775156657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,131071,0.18759999672571817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,1,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,1,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,3,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,3,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,7,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,7,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,15,0.025402667621771496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,15,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,31,0.02500266581773758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,31,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,63,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,63,0.02327999969323476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,127,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,127,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,255,0.024341332415739696
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,255,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,511,0.037802666425704956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,511,0.029258665939172108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,1,0.03155199935038885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,1,0.029109333952267964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,3,0.03134933362404505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,3,0.029359998802344005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,7,0.03186666717131933
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,7,0.029450667401154835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,15,0.03177600105603536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,15,0.02917333443959554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,31,0.03166933357715607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,31,0.029194665451844532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,63,0.031530665854612984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,63,0.029648000995318096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,127,0.031514666974544525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,127,0.029466666281223297
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,255,0.03718933214743932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,255,0.029253333806991577
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,511,0.05769599974155426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,511,0.04164266586303711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,1,0.04977599779764811
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,1,0.04377066592375437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,3,0.04996266464392344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,3,0.04399999976158142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,7,0.05073600014050802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,7,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,15,0.050000001986821495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,15,0.04376000165939331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,31,0.049738665421803795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,31,0.04399466514587402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,63,0.050213331977526345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,63,0.04413333535194397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,127,0.052373334765434265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,127,0.045194665590922035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,255,0.05726933479309082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,255,0.04773866633574168
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,511,0.09661333759625752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,511,0.06621333460013072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,1,0.0867786705493927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,1,0.0745066652695338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,3,0.08714666962623596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,3,0.07499733567237854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,7,0.08674133817354839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,7,0.0746506651242574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,15,0.08680533369382222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,15,0.07441600163777669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,31,0.08678932984670003
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,31,0.07479999959468842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,63,0.08683733145395915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,63,0.07448000212510426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,127,0.08717866738637288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,127,0.07544533411661784
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,255,0.09550933043162028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,255,0.07689066727956136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,511,0.17083199818929037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,511,0.1142080028851827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,1,0.03807999938726425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,1,0.03761066744724909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,3,0.039546666045983635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,3,0.03804266701141993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,7,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,7,0.03771200031042099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,15,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,15,0.03741333385308584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,31,0.039749334255854286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,31,0.03740799923737844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,63,0.037861332297325134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,63,0.03751999884843826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,127,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,127,0.03751466671625773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,255,0.04398400088151296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,255,0.03748800108830134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,1,0.054101333022117615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,1,0.04811733464399973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,3,0.05552533268928528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,3,0.04810666541258494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,7,0.054485330979029335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,7,0.04825599988301595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,15,0.05427733560403188
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,15,0.0480373352766037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,31,0.054197331269582115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,31,0.0481333335240682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,63,0.054144000013669334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,63,0.04816000163555145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,127,0.05611200133959452
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,127,0.0483893354733785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,255,0.06217599908510844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,255,0.051856001218159996
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,1,0.09122666716575623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,1,0.07858666777610779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,3,0.09066133697827657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,3,0.07868266602357228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,7,0.09109333157539368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,7,0.07869333525498708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,15,0.09105599919954936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,15,0.07893333335717519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,31,0.09121066331863403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,31,0.07869333525498708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,63,0.09090666969617207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,63,0.07852800190448761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,127,0.09104532996813457
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,127,0.08126399914423625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,255,0.10059199730555217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,255,0.0827893316745758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,1,0.16059199968973795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,1,0.1397706667582194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,3,0.16062933206558228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,3,0.14012799660364786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,7,0.16060800353686014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,7,0.13974933822949728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,15,0.16060266892115274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,15,0.13980799913406372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,31,0.1606933375199636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,31,0.13877333203951517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,63,0.16053866346677145
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,63,0.1400106648604075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,127,0.1593173344930013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,127,0.13990400234858194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,255,0.17844800154368082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,255,0.14166399836540222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,63,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,511,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1023,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,2047,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,2047,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,4095,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,4095,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,8191,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,8191,0.0200853335360686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,16383,0.027221334477265675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,16383,0.02313599983851115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,32767,0.04154133299986521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,32767,0.02924266705910365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,65535,0.06132266422112783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,65535,0.043552001317342125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,15,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,31,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,127,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,255,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,511,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1023,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,2047,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,2047,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,4095,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,4095,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,8191,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,8191,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,16383,0.03791466603676478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,16383,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,32767,0.0595413347085317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,32767,0.039488000174363456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,65535,0.10121599833170573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,65535,0.060080001751581825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,7,0.07683733105659485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,3,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,31,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,63,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,511,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,511,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1023,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1023,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,2047,0.01783466711640358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,2047,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,4095,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,4095,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,8191,0.037477334340413414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,8191,0.025055999557177227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,16383,0.05964800218741099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,16383,0.03942399968703588
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,32767,0.10122133294741313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,32767,0.06142933170000712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,65535,0.1835626761118571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,65535,0.10336533188819885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,31,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,511,0.01462399959564209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1023,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,2047,0.021061333517233532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,2047,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,4095,0.03757333258787791
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,4095,0.02436800052722295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,8191,0.05958933134873708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,8191,0.039647998909155525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,16383,0.10091200470924377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,16383,0.059061333537101746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,32767,0.18288000424702963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,32767,0.09891200065612793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,65535,0.3498239914576213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,65535,0.1788533329963684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,15,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,63,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,63,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,127,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,511,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1023,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1023,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,2047,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,2047,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,4095,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,4095,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,4095,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,8191,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,8191,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,8191,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,8191,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,16383,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,16383,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,16383,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,16383,0.02309333284695943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,32767,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,32767,0.041573333243529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,32767,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,32767,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,65535,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,65535,0.062421331803003945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,65535,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,65535,0.05420266588528951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,3,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,31,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,63,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,63,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,63,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,255,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,511,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,511,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,511,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1023,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1023,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1023,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,2047,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,2047,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,2047,0.013893333574136099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,2047,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,4095,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,4095,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,4095,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,4095,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,8191,0.013440000514189402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,8191,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,8191,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,8191,0.023306667804718018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,16383,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,16383,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,16383,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,16383,0.03286933402220408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,32767,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,32767,0.06208533545335134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,32767,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,32767,0.05407466491063436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,65535,0.01339200014869372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,65535,0.10733866691589355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,65535,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,65535,0.08913066983222961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,3,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,7,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,15,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,31,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,63,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,127,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,255,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,255,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,511,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1023,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,2047,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,2047,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,2047,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,2047,0.01759999990463257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,4095,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,4095,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,4095,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,4095,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,8191,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,8191,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,8191,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,8191,0.03275199979543686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,16383,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,16383,0.06031466523806254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,16383,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,16383,0.053642665346463524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,32767,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,32767,0.10259733597437541
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,32767,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,32767,0.09036266803741455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,65535,0.014352000008026758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,65535,0.18956265846888223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,65535,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,65535,0.16153066356976828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,7,0.010202666744589806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,7,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,15,0.010048000141978264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,15,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,31,0.010144000252087912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,31,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,63,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,127,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,511,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1023,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1023,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1023,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,2047,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,2047,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,2047,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,2047,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,4095,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,4095,0.011477333803971609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,4095,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,8191,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,8191,0.015413332730531693
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,8191,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,8191,0.015717333803574245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,16383,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,16383,0.020026666422684986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,16383,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,16383,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,32767,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,32767,0.026837334036827087
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,32767,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,32767,0.026154667139053345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,16383,0.060415998101234436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,65535,0.030437332888444264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,65535,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,65535,0.029824001093705494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,131071,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,131071,0.033589333295822144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,131071,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,131071,0.033344000577926636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,3,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,3,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,7,0.01027199998497963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,7,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,7,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,31,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,63,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,127,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,255,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,511,0.010373333469033241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,511,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1023,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1023,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,2047,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,2047,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,2047,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,2047,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,4095,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,4095,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,4095,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,4095,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,8191,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,8191,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,8191,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,8191,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,16383,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,16383,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,16383,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,16383,0.01942933350801468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,32767,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,32767,0.022624000906944275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,32767,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,32767,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,65535,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,65535,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,65535,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,65535,0.02348800003528595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,131071,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,131071,0.030058667063713074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,131071,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,131071,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,3,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,15,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,31,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,31,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,31,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,63,0.010421333213647207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,127,0.010410666465759277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,511,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,511,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,511,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1023,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1023,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,2047,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,2047,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,2047,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,2047,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,4095,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,4095,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,4095,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,4095,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,8191,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,8191,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,8191,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,8191,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,16383,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,16383,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,16383,0.010586666564146677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,16383,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,32767,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,32767,0.020714666694402695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,32767,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,32767,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,65535,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,65535,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,65535,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,131071,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,131071,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,131071,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,131071,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,3,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,15,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,15,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,31,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,31,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,31,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,63,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,127,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,511,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,511,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,511,0.012442667037248611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,511,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1023,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1023,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,2047,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,2047,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,2047,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,2047,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,4095,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,4095,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,4095,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,4095,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,8191,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,8191,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,8191,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,8191,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,16383,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,16383,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,16383,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,16383,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,32767,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,32767,0.02164799968401591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,32767,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,32767,0.021583999196688335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,65535,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,65535,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,65535,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,65535,0.023402666052182514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,131071,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,131071,0.02754666656255722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,131071,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,131071,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,3,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,7,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,7,0.010464000205198923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,7,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,15,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,31,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,31,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,63,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,127,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,255,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,255,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,511,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1023,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1023,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,2047,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,2047,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,2047,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,2047,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,4095,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,4095,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,4095,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,4095,0.013450667262077332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,8191,0.011605333536863327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,8191,0.017509333789348602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,8191,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,8191,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,16383,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,16383,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,16383,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,16383,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,32767,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,32767,0.01969066634774208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,32767,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,32767,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,65535,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,65535,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,65535,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,65535,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,131071,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,131071,0.029631999631722767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,131071,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,131071,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1,0.010501333822806677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,15,0.010570666442314783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,31,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,255,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,511,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,511,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1023,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1023,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,2047,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,2047,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,2047,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,2047,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,4095,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,4095,0.01534933348496755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,4095,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,4095,0.015376000354687372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,8191,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,8191,0.016586666305859882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,8191,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,8191,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,16383,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,16383,0.018112000077962875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,16383,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,16383,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,32767,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,32767,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,32767,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,32767,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,65535,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,65535,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,65535,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,65535,0.023237332701683044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,131071,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,131071,0.04087999959786733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,131071,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,131071,0.03160000095764796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,7,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,7,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,7,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,15,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,15,0.011578666667143503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,31,0.01192533348997434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,31,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,31,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,63,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,63,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,127,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,255,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1023,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1023,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1023,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,2047,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,2047,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,2047,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,4095,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,4095,0.019567999988794327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,4095,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,4095,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,8191,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,8191,0.02516266703605652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,8191,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,8191,0.023226665953795116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,16383,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,16383,0.04182399809360504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,16383,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,16383,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,32767,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,32767,0.062368000547091164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,32767,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,32767,0.05388266841570536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,3,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,3,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,255,0.012186666329701742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,255,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,511,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,511,0.01190399999419848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,511,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1023,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1023,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1023,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,2047,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,2047,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,2047,0.014245333770910898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,2047,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,4095,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,4095,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,4095,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,4095,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,8191,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,8191,0.0396373321612676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,8191,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,8191,0.0341333324710528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,16383,0.014645333091417948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,16383,0.06083733340104421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,16383,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,16383,0.053957333167394005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,32767,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,32767,0.10363200306892395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,32767,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,32767,0.09075733025868733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,3,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,3,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,3,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,3,0.012549333274364471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,7,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,7,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,7,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,15,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,15,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,15,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,15,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,31,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,31,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,31,0.012613333761692047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,31,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,63,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,63,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,63,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,63,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,127,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,127,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,127,0.012608000387748083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,255,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,255,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,255,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,255,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,511,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,511,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,511,0.012666666259368261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,511,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1023,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1023,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1023,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,2047,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,2047,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,2047,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,2047,0.023333333432674408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,4095,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,4095,0.040576001008351646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,4095,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,4095,0.0340639998515447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,8191,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,8191,0.06197333335876465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,8191,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,8191,0.05423999826113383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,16383,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,16383,0.10520000259081523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,16383,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,16383,0.09079466263453166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,32767,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,32767,0.19119999806086221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,32767,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,32767,0.1627679963906606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,3,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,3,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,7,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,15,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,31,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,63,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,63,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,127,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,255,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,255,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,255,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,511,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,511,0.011663999408483505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,511,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1023,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,2047,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,2047,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,2047,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,2047,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,4095,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,4095,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,4095,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,4095,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,8191,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,8191,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,8191,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,8191,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,16383,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,16383,0.019797333826621372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,16383,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,16383,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,32767,0.012703999876976013
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,32767,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,32767,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,32767,0.020549333343903225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,65535,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,65535,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,65535,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,65535,0.022970666488011677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,131071,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,131071,0.03155199935038885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,131071,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,131071,0.029290666182835896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,3,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,3,0.011450666934251785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,3,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,7,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,7,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,15,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,15,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,31,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,127,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,255,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,511,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,511,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,511,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1023,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1023,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,2047,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,2047,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,2047,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,2047,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,4095,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,4095,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,4095,0.01682666689157486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,8191,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,8191,0.01739199956258138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,8191,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,8191,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,16383,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,16383,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,16383,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,16383,0.018805333723624546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,32767,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,32767,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,32767,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,32767,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,65535,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,65535,0.02956799914439519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,65535,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,65535,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,131071,0.014831999937693277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,131071,0.04404800136884054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,131071,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,131071,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,3,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,7,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,7,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,63,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,63,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,511,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1023,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,2047,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,2047,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,2047,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,2047,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,4095,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,4095,0.016997333616018295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,4095,0.01257066677014033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,4095,0.016095999628305435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,8191,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,8191,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,8191,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,8191,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,16383,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,16383,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,16383,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,16383,0.02092266579469045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,32767,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,32767,0.025279998779296875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,32767,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,32767,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,65535,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,65535,0.04159466673930486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,65535,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,65535,0.03310399999221166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,131071,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,131071,0.06458666423956554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,131071,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,131071,0.05562133093674978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,3,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,3,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,7,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,15,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,31,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,63,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,63,0.01257066677014033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,127,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,255,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,255,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,511,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1023,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,2047,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,2047,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,2047,0.01640533283352852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,2047,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,4095,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,4095,0.025557334224383037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,4095,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,4095,0.023386667172114056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,8191,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,8191,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,8191,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,8191,0.03292266776164373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,16383,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,16383,0.06192000210285187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,16383,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,16383,0.053904001911481224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,3,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,3,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,3,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,3,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,7,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,7,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,7,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,15,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,15,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,15,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,15,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,31,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,31,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,31,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,31,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,63,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,63,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,63,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,127,0.012634667257467905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,127,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,127,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,255,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,255,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,255,0.012602667013804117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,511,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,511,0.014458666245142618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1023,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1023,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1023,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1023,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,2047,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,2047,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,4095,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,4095,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,4095,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,4095,0.03349333256483078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,8191,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,8191,0.06286933521429698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,8191,0.01659199967980385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,8191,0.05515199899673462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,16383,0.015450666348139444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,16383,0.10566932956377666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,16383,0.016415999581416447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,16383,0.09130666653315227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,3,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,3,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,7,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,7,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,7,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,7,0.01470400020480156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,15,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,15,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,15,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,15,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,31,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,31,0.015439999600251516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,31,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,31,0.01552533358335495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,63,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,63,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,63,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,63,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,127,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,127,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,127,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,127,0.014570667097965876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,255,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,255,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,255,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,255,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,511,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,511,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,511,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,511,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1023,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1023,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1023,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1023,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,2047,0.017152000218629837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,2047,0.040048000713189445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,2047,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,2047,0.0334346666932106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,4095,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,4095,0.06260266900062561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,4095,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,4095,0.05593599875768026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,8191,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,8191,0.10540800293286641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,8191,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,8191,0.09160000085830688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,16383,0.017664000391960144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,16383,0.19116799036661783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,16383,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,16383,0.16266666849454245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,3,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,3,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,3,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,3,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,7,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,7,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,7,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,7,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,15,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,15,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,15,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,15,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,31,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,31,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,31,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,31,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,63,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,63,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,127,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,127,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,127,0.012469333906968435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,255,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,255,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,255,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,255,0.012597333639860153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,511,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,511,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,511,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,511,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1023,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1023,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1023,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1023,0.016901332885026932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,2047,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,2047,0.023232000569502514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,4095,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,4095,0.04134399940570196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,4095,0.03332266708215078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,3,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,3,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,3,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,7,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,7,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,7,0.014671999961137772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,7,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,15,0.015429332852363586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,15,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,15,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,15,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,31,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,31,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,31,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,31,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,63,0.014607999473810196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,63,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,63,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,63,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,127,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,127,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,127,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,127,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,255,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,255,0.014549333602190018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,255,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,255,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,511,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,511,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,511,0.01743999992807706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1023,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1023,0.023413332800070446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1023,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1023,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,2047,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,2047,0.04126933217048645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,2047,0.017514667163292568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,2047,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,4095,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,4095,0.063509335120519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,4095,0.018661333868900936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,4095,0.055786664287249245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,3,0.018538666268189747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,3,0.018629333625237148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,3,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,3,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,7,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,7,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,7,0.018746666610240936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,7,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,15,0.018976000448067982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,15,0.018191999445358913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,15,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,15,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,31,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,31,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,31,0.01785600061217944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,31,0.018672000616788864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,63,0.01884799947341283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,63,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,63,0.018309333672126133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,63,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,127,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,127,0.01752000053723653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,127,0.017445333302021027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,127,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,255,0.018559999763965607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,255,0.018144000321626663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,255,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,255,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,511,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,511,0.023386667172114056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,511,0.01869333287080129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,511,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1023,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1023,0.03559466699759165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1023,0.018853332847356796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1023,0.031504000226656594
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,2047,0.02126399924357732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,2047,0.058149332801500954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,2047,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,2047,0.05206400156021118
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,4095,0.021386665602525074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,4095,0.0972160001595815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,4095,0.021482666333516438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,4095,0.08503466844558716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,3,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,3,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,3,0.014682666709025701
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,3,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,7,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,7,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,7,0.014826666563749313
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,7,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,15,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,15,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,15,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,15,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,31,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,31,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,31,0.014576000471909841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,31,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,63,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,63,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,63,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,63,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,127,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,127,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,127,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,127,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,255,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,255,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,255,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,255,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,511,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,511,0.01889066646496455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,511,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,511,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1023,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1023,0.02348800003528595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1023,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1023,0.022554665803909302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,2047,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,2047,0.04151466737190882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,2047,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,2047,0.033344000577926636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1,0.01878400022784869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,3,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,3,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,3,0.019178666174411774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,3,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,7,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,7,0.018735999862353008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,7,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,15,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,15,0.018842666099468868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,15,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,15,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,31,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,31,0.01924266666173935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,31,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,31,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,63,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,63,0.018885333091020584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,63,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,63,0.017263999829689663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,127,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,127,0.01870399961868922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,127,0.017621333400408428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,127,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,255,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,255,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,255,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,255,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,511,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,511,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,511,0.018794666975736618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,511,0.023205332458019257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1023,0.01923199991385142
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1023,0.03558400024970373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1023,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1023,0.03035733352104823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,2047,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,2047,0.05898133416970571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,2047,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,2047,0.051829333106676735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1,0.025733334322770435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,3,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,3,0.025754667818546295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,3,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,3,0.02534399926662445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,7,0.02701333413521449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,7,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,7,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,7,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,15,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,15,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,15,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,15,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,31,0.027285332481066387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,31,0.025418666501839954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,31,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,31,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,63,0.025568000972270966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,63,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,63,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,63,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,127,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,127,0.027530667682488758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,127,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,127,0.025888000925381977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,255,0.027701333165168762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,255,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,255,0.025818665822347004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,255,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,511,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,511,0.039936001102129616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,511,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,511,0.03348266581694285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1023,0.0271519993742307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1023,0.05806399881839752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1023,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1023,0.052426666021347046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,2047,0.029605334003766377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,2047,0.09938666224479675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,2047,0.029359998802344005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,2047,0.0869706670443217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1,0.010543999572594961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,3,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,15,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,63,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,127,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,511,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,2047,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,2047,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,4095,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,4095,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,4095,0.01691199963291486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,8191,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,8191,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,8191,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,8191,0.017903999735911686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,16383,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,16383,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,16383,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,16383,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,32767,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,32767,0.021568000316619873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,32767,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,32767,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,65535,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,65535,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,65535,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,65535,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,131071,0.015333333363135656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,131071,0.04178133110205332
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,131071,0.014890667051076889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,131071,0.03367999941110611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,3,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,7,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,7,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,15,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,15,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,63,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,63,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,63,0.011653333902359009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,63,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,127,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,255,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,255,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,255,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,511,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,511,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1023,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,2047,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,2047,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,4095,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,4095,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,4095,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,4095,0.016783999900023144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,8191,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,8191,0.01887999971707662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,8191,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,8191,0.017290666699409485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,16383,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,16383,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,16383,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,16383,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,32767,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,32767,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,32767,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,32767,0.025013332565625507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,65535,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,65535,0.041690667470296226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,65535,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,65535,0.03294933338960012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,131071,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,131071,0.06276266773541768
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,131071,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,131071,0.05435200035572052
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,3,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,3,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,7,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,7,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,7,0.011461333682139715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,15,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,15,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,31,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,127,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,255,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,255,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,511,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,511,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,511,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1023,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1023,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,2047,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,2047,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,2047,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,4095,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,4095,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,4095,0.016970666746298473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,8191,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,8191,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,8191,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,8191,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,16383,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,16383,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,16383,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,16383,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,32767,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,32767,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,32767,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,32767,0.031354665756225586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,65535,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,65535,0.0624533345301946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,65535,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,65535,0.05358933409055074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,131071,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,131071,0.10900266965230306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,131071,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,131071,0.09109333157539368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1,0.019509332875410717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,3,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,3,0.019754666835069656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,3,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,3,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,7,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,7,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,7,0.0170666662355264
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,7,0.017557332913080852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,15,0.019482667247454327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,15,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,15,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,15,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,31,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,31,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,31,0.01754133279124896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,31,0.017893332988023758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,63,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,63,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,63,0.018624000251293182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,63,0.018245333184798557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,127,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,127,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,127,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,127,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,255,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,255,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,255,0.01932266727089882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,255,0.018751999984184902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,511,0.0191040001809597
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,511,0.024933333198229473
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,511,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,511,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1023,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1023,0.035690667728583016
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1023,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1023,0.03146666785081228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1,0.025472000241279602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1,0.025626666843891144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,3,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,3,0.027349332968393963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,3,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,3,0.02554133286078771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,7,0.027386667827765148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,7,0.027056001126766205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,7,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,7,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,15,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,15,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,15,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,15,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,31,0.027424000203609467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,31,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,31,0.02517866591612498
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,31,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,63,0.027119999130566914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,63,0.027317332724730175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,63,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,63,0.025093334416548412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,127,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,127,0.02589333305756251
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,127,0.025594666600227356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,127,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,255,0.027477333943049114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,255,0.027210667729377747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,255,0.027248000105222065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,255,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,511,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,511,0.03982399900754293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,511,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,511,0.035674666364987694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1023,0.02743999908367793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1023,0.05851200222969055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1023,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1023,0.05273066461086273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1,0.04207466542720795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1,0.03957333415746689
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1,0.03961066653331121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,3,0.04192000130812327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,3,0.041984001795450844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,3,0.03949866692225138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,3,0.03939733405907949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,7,0.041690667470296226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,7,0.04188266893227895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,7,0.03967999915281931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,7,0.039877332746982574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,15,0.0422986646493276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,15,0.04264533519744873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,15,0.03989866624275843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,15,0.03979199876387914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,31,0.04191466669241587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,31,0.04181866844495138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,31,0.03940266619126002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,31,0.03965866565704346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,63,0.04199466605981191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,63,0.04221866528193156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,63,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,63,0.03990933299064636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,127,0.04189866781234741
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,127,0.04197866717974345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,127,0.03976000100374222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,127,0.03982399900754293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,255,0.043840001026789345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,255,0.04384533564249674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,255,0.039690665900707245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,255,0.0395359992980957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,511,0.044122666120529175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,511,0.06481599807739258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,511,0.04170133173465729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,511,0.058677335580190025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1023,0.043866669138272606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1023,0.10150933265686035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1023,0.04166933397452036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1023,0.08945600191752116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,1,0.027615999182065327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,1,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,1,0.02535466601451238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,1,0.025727999707063038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,3,0.02749866743882497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,3,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,3,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,3,0.025434667865435284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,7,0.027727998793125153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,7,0.027632000545660656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,7,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,7,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,15,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,15,0.027514666318893433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,15,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,15,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,31,0.027450665831565857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,31,0.0276053324341774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,31,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,31,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,63,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,63,0.027802666028340656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,63,0.027093333502610523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,63,0.0271573339899381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,127,0.027290667096773785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,127,0.0277813325325648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,127,0.025263999899228413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,65535,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,127,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,255,0.02752533306678136
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,255,0.027402666707833607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,255,0.02693866689999898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,511,0.029152000943819683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,255,0.028416000306606293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,511,0.04053866614898046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,511,0.02779199928045273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,511,0.035749333600203194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,1,0.04366933306058248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,1,0.043807998299598694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,1,0.039962666730086006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,1,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,3,0.04371733466784159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,3,0.0436106671889623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,3,0.039861333866914116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,3,0.03972266614437103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,7,0.04350399971008301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,7,0.043552001317342125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,7,0.03992533435424169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,7,0.03969600051641464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,15,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,15,0.04369066655635834
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,15,0.03953066716591517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,15,0.03993066648642222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,31,0.04386133452256521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,31,0.04359999795754751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,65535,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,31,0.03985599925120672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,31,0.040149333576361336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,63,0.04353066782156626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,63,0.0436160018046697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,63,0.04012266546487808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,63,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,127,0.044250667095184326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,127,0.03997333347797394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,127,0.04018666595220566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,255,0.04427733520666758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,255,0.04502399762471517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,255,0.04144533226887385
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,127,0.04378666480382284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,255,0.0400693342089653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,511,0.04367466767628988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,511,0.06660800178845723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,511,0.0418453315893809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,511,0.06009600063165029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,1,0.07461333274841309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,1,0.07474133372306824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,1,0.06889600058396657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,1,0.06838933130105336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,3,0.07464533547560374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,3,0.07488533357779185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,3,0.06868266562620799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,3,0.06859733164310455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,7,0.07428800066312154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,7,0.06864533325036366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,7,0.06832000116507213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,15,0.0747519979874293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,15,0.07463466624418895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,15,0.06869866450627644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,15,0.06834666430950165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,31,0.07495999832948048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,31,0.07463466624418895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,31,0.06856533388296764
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,31,0.06868800024191539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,63,0.07483733197053273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,63,0.06866666674613953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,63,0.07461333274841309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,63,0.06836266815662384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,127,0.07649066547552745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,127,0.07678933441638947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,127,0.06855999926726024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,127,0.06844266752401988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,255,0.07766399780909221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,255,0.07658133407433827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,255,0.07122133175532024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,255,0.07041066884994507
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,511,0.0784746656815211
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,511,0.11654399832089742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,511,0.07047999898592631
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,3,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,511,0.10524800419807434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,3,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,7,0.01239466667175293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,7,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,15,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,15,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,31,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,63,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,63,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1023,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1023,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1023,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,2047,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,2047,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,2047,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,2047,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,4095,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,4095,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,4095,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,8191,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,8191,0.019093333433071773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,8191,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,8191,0.018757333358128864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,16383,0.01351999988158544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,16383,0.021344001094500225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,16383,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,16383,0.02033599962790807
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,32767,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,32767,0.026186667382717133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,32767,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,32767,0.024234667420387268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,65535,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,65535,0.0414986660083135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,65535,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,65535,0.03310399999221166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,131071,0.015008000036080679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,131071,0.06243733565012614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,131071,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,7,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,7,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,7,0.01032533310353756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,7,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,15,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,63,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,127,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,511,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,511,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1023,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,2047,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,2047,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,2047,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,2047,0.015114666273196539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,4095,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,4095,0.01687466725707054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,4095,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,4095,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,8191,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,8191,0.019280000279347103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,8191,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,8191,0.020634666085243225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,16383,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,16383,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,16383,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,16383,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,32767,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,32767,0.04118400067090988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,32767,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,32767,0.031871999303499855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,65535,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,65535,0.06259199976921082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,65535,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,65535,0.05418666700522105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,131071,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,131071,0.11023466785748799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,131071,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,131071,0.09149332841237386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,3,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,3,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,7,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,7,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,7,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,15,0.011472000430027643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,15,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,15,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,31,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,31,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,63,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,63,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,63,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,63,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,127,0.01157333329319954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,127,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,255,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,511,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,511,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1023,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1023,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1023,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1023,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,2047,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,2047,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,2047,0.015562667200962702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,4095,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,4095,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,4095,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,8191,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,8191,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,8191,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,8191,0.023408000667889912
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,16383,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,16383,0.03937600056330363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,16383,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,16383,0.03159466634194056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,32767,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,32767,0.061749334136645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,32767,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,32767,0.053861334919929504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,65535,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,65535,0.10542933146158855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,65535,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,65535,0.08916266759236653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,131071,0.015615999698638916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,131071,0.19754133621851602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,131071,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,131071,0.16201066970825195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,3,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,15,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,255,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,511,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,511,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1023,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,2047,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,2047,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,4095,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,4095,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,8191,0.023567999402681988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,8191,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,16383,0.040250666439533234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,16383,0.025941332181294758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,32767,0.06196799874305725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,32767,0.04134399940570196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,65535,0.1034346620241801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,65535,0.06085866689682007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,7,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,255,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,511,0.013546666751305262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,511,0.013408000270525614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1023,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1023,0.013487999637921652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,2047,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,2047,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,4095,0.021840001145998638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,4095,0.019493332753578823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,8191,0.04020266731580099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,8191,0.025407999753952026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,16383,0.06018133461475372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,16383,0.04019733270009359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,32767,0.10178132851918538
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,32767,0.06149866680304209
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,65535,0.18372799952824911
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,65535,0.10291199882825215
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,3,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,3,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,7,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,15,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,31,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,127,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1023,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1023,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,2047,0.021114667256673176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,2047,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,4095,0.037690666814645134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,4095,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,8191,0.059994667768478394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,8191,0.039706667264302574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,16383,0.1011199951171875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,16383,0.06055466830730438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,32767,0.1822026570638021
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,32767,0.10054933031400044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,65535,0.34517868359883624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,65535,0.17893334229787192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,31,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,63,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,127,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,511,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,511,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1023,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,2047,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,2047,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,4095,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,4095,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,8191,0.016917333006858826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,8191,0.016832000265518825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,16383,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,16383,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,32767,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,32767,0.027130665878454845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,65535,0.03156800071398417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,65535,0.03160533308982849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,131071,0.035786665976047516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,131071,0.033973333736260734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,7,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,31,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,127,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,255,0.010255999863147736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,511,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,511,0.011877333124478659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1023,0.011733333269755045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1023,0.011749333391586939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,2047,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,4095,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,4095,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,8191,0.0173333336909612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,8191,0.015578666081031164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,16383,0.019744000087181728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,16383,0.020117333779732387
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,32767,0.023631999890009563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,32767,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,65535,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,65535,0.02532800038655599
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,131071,0.0321066677570343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,131071,0.029674666623274486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,3,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,3,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,7,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,31,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,127,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,511,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,511,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1023,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1023,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,2047,0.012773333738247553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,2047,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,4095,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,8191,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,8191,0.017498667041460674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,16383,0.020693333198626835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,16383,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,32767,0.0215786670645078
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,32767,0.02089600016673406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,65535,0.025360000630219776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,65535,0.02276800076166789
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,131071,0.04307733476161957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,131071,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,15,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,31,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,255,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1023,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,2047,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,2047,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,4095,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,8191,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,8191,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,16383,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,16383,0.021253332495689392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,32767,0.024314666787783306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,32767,0.023130667706330616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,65535,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,65535,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,131071,0.031717332700888314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,131071,0.029674666623274486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,7,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,63,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,127,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,255,0.011866666376590729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1023,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1023,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,2047,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,2047,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,4095,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,8191,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,8191,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,16383,0.020394666741291683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,16383,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,32767,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,32767,0.02111999938885371
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,65535,0.025439999997615814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,65535,0.02364266663789749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,131071,0.04398400088151296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,131071,0.031898667414983116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,15,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,255,0.0103946669648091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,511,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,511,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1023,0.011989332735538483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1023,0.01179733375708262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,2047,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,2047,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,4095,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,4095,0.01669866715868314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,8191,0.017850667238235474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,8191,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,16383,0.01951466624935468
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,16383,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,32767,0.02359466751416524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,32767,0.021317332983016968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,65535,0.03945599993069967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,65535,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,131071,0.06087466577688853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,131071,0.040720000863075256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,511,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,511,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1023,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1023,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,2047,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,2047,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,4095,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,4095,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,8191,0.04067733387152354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,8191,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,16383,0.06230400005976359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,16383,0.040234667559464775
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,32767,0.10281599561373393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,32767,0.06108266611893972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,3,0.011898666620254517
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,3,0.011637333780527115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,7,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,31,0.011530666301647821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,511,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,511,0.01368533323208491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1023,0.01598400001724561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1023,0.014629332969586054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,2047,0.02216533323129018
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,2047,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,4095,0.039706667264302574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,4095,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,8191,0.060319999853769936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,8191,0.039962666730086006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,16383,0.1013866662979126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,16383,0.05898133416970571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,32767,0.1824373404184977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,32767,0.10068266590436299
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,3,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,3,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,7,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,15,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,15,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,31,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,31,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,63,0.013455999394257864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,63,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,127,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,255,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,255,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,511,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,511,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1023,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1023,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,2047,0.0390079990029335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,2047,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,4095,0.060122668743133545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,4095,0.04070399949947993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,8191,0.10178666313489278
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,8191,0.062037333846092224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,16383,0.18494399388631186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,16383,0.10282133022944133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,32767,0.34913067022959393
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,32767,0.1849600076675415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,3,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,7,0.010319999729593595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,63,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,127,0.010640000303586325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,511,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1023,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1023,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,2047,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,2047,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,4095,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,4095,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,8191,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,8191,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,16383,0.02144533395767212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,16383,0.02086399992307027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,32767,0.023546665906906128
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,32767,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,65535,0.02739199995994568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,65535,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,131071,0.04604266583919525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,131071,0.03179199993610382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,7,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,15,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1023,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1023,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,2047,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,4095,0.01721599946419398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,4095,0.017504000415404636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,8191,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,8191,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,16383,0.021850667893886566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,16383,0.020768000433842342
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,32767,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,32767,0.02319466571013133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,65535,0.04171733558177948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,65535,0.027130665878454845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,131071,0.06345599889755249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,131071,0.04438399771849314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,3,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,511,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,511,0.012432000289360682
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1023,0.012096000214417776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,2047,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,2047,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,4095,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,4095,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,8191,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,8191,0.018922666708628338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,16383,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,16383,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,32767,0.03975466638803482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,32767,0.02737066646416982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,65535,0.0602453351020813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,65535,0.04191466669241587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,131071,0.1018453339735667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,131071,0.06230400005976359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,3,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,3,0.012714666624863943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,7,0.012250666817029318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,7,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,15,0.012181332955757776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,15,0.012576000144084295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,31,0.01198400060335795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,31,0.012026666353146235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,63,0.012410666793584824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,127,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,511,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,511,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1023,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1023,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,2047,0.021189334491888683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,2047,0.019120000302791595
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,4095,0.040133332212766014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,4095,0.025439999997615814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,8191,0.06145599981149038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,8191,0.04014399896065394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,16383,0.1021066705385844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,16383,0.06006399790445963
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,3,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,3,0.012586666891972223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,7,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,7,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,15,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,15,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,31,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,31,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,63,0.012655999511480331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,63,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,127,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,255,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,255,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,511,0.015237333873907724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,511,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1023,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1023,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,2047,0.04002666721741358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,2047,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,4095,0.060453335444132485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,4095,0.041834667325019836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,8191,0.1034346620241801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,8191,0.06197333335876465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,16383,0.18466132879257202
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,16383,0.1037546694278717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,3,0.016762666404247284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,7,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,7,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,15,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,15,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,31,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,31,0.015279999623696009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,63,0.016469333320856094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,63,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,127,0.015392000476519266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,127,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,255,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,255,0.01522133375207583
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,511,0.019445333629846573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,511,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1023,0.033957332372665405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1023,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,2047,0.06122133135795593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,2047,0.04186133543650309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,4095,0.1030506690343221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,4095,0.06346666812896729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,8191,0.18530132373174033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,8191,0.10479999581972758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,16383,0.35074134667714435
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,16383,0.18736533323923746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,3,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,3,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,7,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,7,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,15,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,15,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,31,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,31,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,63,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,127,0.015077333897352219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,127,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,255,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,255,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,511,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,511,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1023,0.02053333322207133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1023,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,2047,0.04073066761096319
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,2047,0.02624000112215678
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,4095,0.06311999758084615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,4095,0.039808000127474465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,3,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,3,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,7,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,7,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,15,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,15,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,31,0.01648533344268799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,31,0.015520000209410986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,63,0.016778666526079178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,63,0.01524266724785169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,127,0.01605333387851715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,127,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,255,0.015557333827018738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,255,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,511,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,511,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1023,0.03472000112136205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1023,0.023381332556406658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,2047,0.062122667829195656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,2047,0.04160533348719279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,4095,0.10381866494814555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,4095,0.062218666076660156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1,0.01937066639463107
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,3,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,3,0.019066666563351948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,7,0.019258666783571243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,7,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,15,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,15,0.01876266673207283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,31,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,31,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,63,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,63,0.018944000204404194
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,127,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,127,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,255,0.020981334149837494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,255,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,511,0.03386666625738144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,511,0.023215999205907185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1023,0.05229333539803823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1023,0.035455999275048576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,2047,0.09582933783531189
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,2047,0.05827199916044871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,4095,0.1704639991124471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,4095,0.09693866968154907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,3,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,3,0.014885333677132925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,7,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,7,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,15,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,15,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,31,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,31,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,63,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,63,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,127,0.017162666966517765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,127,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,255,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,255,0.015285332997639975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,511,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,511,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1023,0.03513599932193756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1023,0.02457600086927414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,2047,0.06182933350404104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,2047,0.04141866664091746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1,0.019487999379634857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1,0.0189280000825723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,3,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,3,0.019333332777023315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,7,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,7,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,15,0.020986666282018025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,15,0.019189332922299702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,31,0.02109866589307785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,31,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,63,0.02085866779088974
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,63,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,127,0.021013334393501282
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,127,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,255,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,255,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,511,0.033626665671666466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,511,0.025397333006064098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1023,0.05415999889373779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1023,0.03554133325815201
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,2047,0.09749333063761394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,2047,0.059338668982187905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1,0.02957333376010259
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,3,0.02923733244339625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,3,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,7,0.02922133356332779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,7,0.027258666853109997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,15,0.029498666524887085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,15,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,31,0.029525332152843475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,31,0.027072000006834667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,63,0.02962133288383484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,63,0.025461333493391674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,127,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,127,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,255,0.03359466542800268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,255,0.027471999327341717
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,511,0.05409599840641022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,511,0.0379573330283165
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1023,0.0918933351834615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1023,0.05704533557097117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,2047,0.17287999391555786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,2047,0.09884267052014668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,31,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1023,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,2047,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,2047,0.015109332899252573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,4095,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,4095,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,8191,0.019440000255902607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,8191,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,16383,0.02161066730817159
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,16383,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,32767,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,32767,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,65535,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,65535,0.027237333357334137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,131071,0.062394668658574425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,131071,0.04435733457406362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,7,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,63,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,127,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,511,0.0124746672809124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,511,0.012719999998807907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1023,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1023,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,2047,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,4095,0.01714666684468587
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,4095,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,8191,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,8191,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,16383,0.0236160010099411
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,16383,0.021312000850836437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,32767,0.04044266790151596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,32767,0.027109332382678986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,65535,0.060831998785336815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,65535,0.0407679999868075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,131071,0.10069866975148518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,131071,0.06252266466617584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,3,0.011514666179815928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,15,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,63,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,511,0.012597333639860153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,511,0.01146666705608368
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1023,0.012144000579913458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1023,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,2047,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,2047,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,4095,0.017423999806245167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,4095,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,8191,0.021984001000722248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,8191,0.019317333896954853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,16383,0.0391893337170283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,16383,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,32767,0.059258664647738137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,32767,0.039605334401130676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,65535,0.10019733508427937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,65535,0.06041066845258077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,131071,0.18416533867518106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,131071,0.10548800230026245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1,0.021338666478792827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,3,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,3,0.019578666736682255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,7,0.021301334102948506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,7,0.01940800001223882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,15,0.02142400046189626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,15,0.019498666127522785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,31,0.02147199958562851
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,31,0.019013332823912304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,63,0.0210506667693456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,63,0.019274666905403137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,127,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,127,0.019173332800467808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,255,0.021210665504137676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,255,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,511,0.03569599986076355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,511,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1023,0.0557226687669754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1023,0.037685332198937736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1,0.029440000653266907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1,0.027445333699385326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,3,0.03049066662788391
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,3,0.02685333291689555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,7,0.029557332396507263
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,7,0.02754133443037669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,15,0.029696000119050343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,15,0.0273333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,31,0.029370665550231934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,31,0.027077332139015198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,63,0.02979733298222224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,63,0.027808000644048054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,127,0.029663999875386555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,127,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,255,0.033743999898433685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,255,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,511,0.05461333195368449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,511,0.03961066653331121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1023,0.09318400422732036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1023,0.058133333921432495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1,0.0481279989083608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1,0.041797334949175514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,3,0.048207998275756836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,3,0.04203199843565623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,7,0.04790933430194855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,7,0.041738669077555336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,15,0.04790933430194855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,15,0.041706666350364685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,31,0.04798933366934458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,31,0.04181866844495138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,63,0.047781333327293396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,63,0.04152533411979675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,127,0.04969066878159841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,127,0.04194133480389913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,255,0.05402133365472158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,255,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,511,0.09351999560991923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,511,0.06442666550477345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1023,0.1686826745669047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1023,0.09893332918485005
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,1,0.03194133440653483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,1,0.029296000798543293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,3,0.03156800071398417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,3,0.02992533395687739
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,7,0.03324799984693527
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,7,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,15,0.03380800038576126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,15,0.029781334102153778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,31,0.03362133353948593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,31,0.030239999294281006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,63,0.03312533348798752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,63,0.029861333469549816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,127,0.031983998914559685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,127,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,255,0.035989334185918175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,255,0.029317334294319153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,511,0.05826666454474131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,511,0.04186666508515676
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,1,0.050111999114354454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,1,0.043935999274253845
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,3,0.05027199784914652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,3,0.044138665000597634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,7,0.05072533090909322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,7,0.044079999128977455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,15,0.050106664498647056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,15,0.043696001172065735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,31,0.050101334849993386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,31,0.04381333291530609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,63,0.05063466727733612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,63,0.04393066465854645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,127,0.052000001072883606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,127,0.04409599800904592
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,255,0.05771199862162272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,255,0.046069333950678505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,511,0.0967733363310496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,511,0.06645866731802623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,1,0.08729599912961324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,1,0.07445333401362102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,3,0.08705066641171773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,3,0.07464000085989635
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,7,0.08698667089144389
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,7,0.07447466750939687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,15,0.08681066830952962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,15,0.07513600091139476
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,31,0.08714133501052856
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1,0.04243200023969015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,31,0.07477866609891255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,63,0.0870293378829956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,63,0.07431999842325847
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,127,0.08703466256459554
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,127,0.0766186664501826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,255,0.09588266412417094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,255,0.078015998005867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,511,0.17151467005411783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,511,0.11575466394424438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,511,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1023,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1023,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,2047,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,2047,0.01672533278663953
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,4095,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,4095,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,8191,0.021370666722456615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,8191,0.019023999571800232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,16383,0.023391999304294586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,16383,0.021359999974568684
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,32767,0.04001066585381826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,32767,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,65535,0.06127466758092245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,65535,0.04101333270470301
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,131071,0.10340799887975057
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,131071,0.06214933097362518
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,7,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,31,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,31,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,63,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,255,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,511,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,511,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1023,0.012133333832025528
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,2047,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,4095,0.01740266631046931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,4095,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,8191,0.023189333577950794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,16383,0.0397173340121905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,16383,0.025610665480295818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,32767,0.060362666845321655
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,32767,0.0399893323580424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,65535,0.10252267122268677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,65535,0.06101333101590475
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,131071,0.18728532393773398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,131071,0.1055573324362437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,7,0.0116799995303154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,15,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,255,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,511,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,511,0.012442667037248611
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,2047,0.019007999449968338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,2047,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,4095,0.02311466634273529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,4095,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,8191,0.03800000001986822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,8191,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,16383,0.05932266513506571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,16383,0.04011200120051702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,32767,0.10108799735705058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,32767,0.06058666606744131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,65535,0.1829599936803182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,65535,0.10180266698201497
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,131071,0.3500853379567464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,131071,0.18775999546051025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1,0.011413333316644033
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,3,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,3,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,7,0.010538666198650995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,7,0.011418666690587997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,15,0.012005332857370377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,31,0.012069333344697952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,31,0.011781333635250727
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,31,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,63,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,127,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,255,0.01184533288081487
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1023,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1023,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1023,0.01138666644692421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,2047,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,2047,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,2047,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,2047,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,4095,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,4095,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,4095,0.014266667266686758
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,4095,0.01718933383623759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,8191,0.013274667163689932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,8191,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,8191,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,8191,0.019296000401178997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,16383,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,16383,0.025706666211287182
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,16383,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,16383,0.02329600105683009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,32767,0.01312000056107839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,32767,0.04002666721741358
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,32767,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,32767,0.031632001201311745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,65535,0.013343999783198038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,65535,0.06271466612815857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,65535,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,65535,0.05433600147565206
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,131071,0.01571200042963028
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,131071,0.11148800452550252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,131071,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,131071,0.09248000383377075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,3,0.011391999820868174
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,15,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,31,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,31,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,127,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,127,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,127,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,255,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,511,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,511,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1023,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1023,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,2047,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,2047,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,2047,0.014127999544143677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,2047,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,4095,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,4095,0.019519999623298645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,8191,0.013616000612576803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,8191,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,8191,0.014218666901191076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,8191,0.023610666394233704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,16383,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,16383,0.04050666590531667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,16383,0.013679999858140945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,16383,0.033333333830038704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,32767,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,32767,0.0641653339068095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,32767,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,32767,0.05240533252557119
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,65535,0.013674666484196981
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,65535,0.1072746713956197
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,65535,0.013280000537633896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,65535,0.08906132976214091
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,131071,0.015642666568358738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,131071,0.19792000452677408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,131071,0.015487999965747198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,131071,0.16327466567357382
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,3,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,3,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,7,0.07508799930413564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,3,0.01044800008336703
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,7,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,7,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,7,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,15,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,63,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,127,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,127,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,127,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,127,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,255,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,255,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1023,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,2047,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,2047,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,2047,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,2047,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,4095,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,4095,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,4095,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,4095,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,8191,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,8191,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,8191,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,8191,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,16383,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,16383,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,16383,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,16383,0.018858666221300762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,32767,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,32767,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,32767,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,32767,0.025450666745503742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,65535,0.029792000850041706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,65535,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,65535,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,65535,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,131071,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,131071,0.03384000062942505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,131071,0.012618667135636011
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,131071,0.033386667569478355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1,0.010581333190202713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,3,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,3,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,7,0.010485333700974783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,7,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,15,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,15,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,63,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,63,0.01062400018175443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,63,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,127,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,255,0.010527999450763067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,255,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,255,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,511,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,511,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1023,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1023,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1023,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1023,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,2047,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,2047,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,2047,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,2047,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,4095,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,4095,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,4095,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,4095,0.011706666400035223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,8191,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,8191,0.01594666639963786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,8191,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,8191,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,16383,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,16383,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,16383,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,16383,0.02091199904680252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,32767,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,32767,0.022458667556444805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,32767,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,32767,0.023445333043734234
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,65535,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,65535,0.024010665714740753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,65535,0.010490667074918747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,65535,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,131071,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,131071,0.02808533360560735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,131071,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,3,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,3,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,7,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,15,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,15,0.0106133334338665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,15,0.010597333312034607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,63,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,63,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,127,0.010410666465759277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,511,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,511,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,511,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1023,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1023,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,2047,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,2047,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,2047,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,2047,0.011861333002646765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,4095,0.01181866725285848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,4095,0.013781332721312841
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,4095,0.011541333049535751
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,4095,0.013477332890033722
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,8191,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,8191,0.015509333461523056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,131071,0.05429866909980774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,8191,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,8191,0.015365333606799444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,16383,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,16383,0.02107733239730199
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,16383,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,16383,0.021322667598724365
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,32767,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,32767,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,32767,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,32767,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,65535,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,65535,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,65535,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,65535,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,131071,0.013349333157142004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,131071,0.02943466603755951
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,131071,0.013359999905029932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,131071,0.029274667302767437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,7,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,7,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,15,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,15,0.009125333279371262
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,15,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,31,0.008879999940594038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,63,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,63,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,127,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,127,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,255,0.009392000113924345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,511,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,511,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,511,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1023,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1023,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1023,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1023,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,2047,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,2047,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,2047,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,2047,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,4095,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,4095,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,4095,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,4095,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,8191,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,8191,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,8191,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,8191,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,16383,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,16383,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,16383,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,16383,0.01905599981546402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,32767,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,32767,0.021007999777793884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,32767,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,32767,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,65535,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,65535,0.02314666658639908
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,65535,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,65535,0.023200000325838726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,131071,0.013461332768201828
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,131071,0.02926933268706004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,131071,0.01341333364446958
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,131071,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,3,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,3,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,7,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,31,0.011503999431928
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,31,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,63,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,255,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,255,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,511,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,511,0.011882666498422623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1023,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1023,0.013701333353916803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,2047,0.013023999830087027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,2047,0.01720533271630605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,2047,0.01451733335852623
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,2047,0.016645333419243496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,4095,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,4095,0.019354666272799175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,4095,0.013839999834696451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,4095,0.019098666807015736
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,8191,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,8191,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,8191,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,8191,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,16383,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,16383,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,16383,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,16383,0.03364266703526179
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,32767,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,32767,0.06373333434263866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,32767,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,32767,0.05389333268006643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,65535,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,65535,0.10790399710337321
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,65535,0.013253333667914072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,65535,0.0904266635576884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,31,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,31,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,63,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,127,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,127,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,255,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,511,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,511,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1023,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1023,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,2047,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,2047,0.019152000546455383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,2047,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,2047,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,4095,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,4095,0.025487999121348064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,4095,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,4095,0.023418667415777843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,8191,0.014896000425020853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,8191,0.03976533313592275
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,8191,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,8191,0.03156266609827677
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,16383,0.013242666920026144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,16383,0.06029333174228668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,16383,0.053360000252723694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,16383,0.015370666980743408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,32767,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,32767,0.10457600156466167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,32767,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,32767,0.09046399593353271
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,65535,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,65535,0.18915732701619467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,65535,0.015344000111023584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,65535,0.1628106633822123
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,3,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,3,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,7,0.010426666587591171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,7,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,15,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,31,0.01009599988659223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,31,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,63,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,63,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,127,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,127,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,255,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,255,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,511,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,511,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1023,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1023,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1023,0.010677333921194077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1023,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,2047,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,2047,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,2047,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,4095,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,4095,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,4095,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,4095,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,8191,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,8191,0.018272000054518383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,8191,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,8191,0.01874133323629697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,16383,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,16383,0.019365333020687103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,16383,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,16383,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,32767,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,32767,0.021840001145998638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,32767,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,32767,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,65535,0.012191999703645706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,65535,0.02502399931351344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,65535,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,65535,0.02362666775782903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,131071,0.013418667018413544
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,131071,0.03152533372243246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,131071,0.015504000087579092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,131071,0.029685333371162415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,7,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,7,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,7,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,31,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,255,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,511,0.010751999914646149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,511,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,511,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1023,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1023,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1023,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1023,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,2047,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,2047,0.013210666676362356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,2047,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,2047,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,4095,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,4095,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,4095,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,4095,0.015893333901961643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,8191,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,8191,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,8191,0.017840000490347546
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,8191,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,16383,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,16383,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,16383,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,16383,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,32767,0.012693333129088083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,32767,0.02123733361562093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,32767,0.021082667013009388
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,65535,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,65535,0.027141332626342773
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,65535,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,65535,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,131071,0.015130666395028433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,131071,0.043680002291997276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,131071,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,131071,0.0346666673819224
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,3,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,3,0.011557333171367645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,7,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,15,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,63,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,511,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,511,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,511,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1023,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1023,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1023,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,2047,0.013232000172138214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,2047,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,2047,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,2047,0.019546666493018467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,4095,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,4095,0.025765334566434223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,4095,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,4095,0.023290666441122692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,8191,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,8191,0.039861333866914116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,8191,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,8191,0.03294933338960012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,16383,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,16383,0.062234664956728615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,16383,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,16383,0.053786665201187134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,32767,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,32767,0.10459733009338379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,32767,0.014277332772811254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,32767,0.09081600109736125
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,3,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,3,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,3,0.012698666503032049
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,3,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,7,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,7,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,7,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,15,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,15,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,63,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,63,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,63,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,63,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,127,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,127,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,127,0.01145600030819575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,255,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,255,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,255,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,511,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,511,0.01488000030318896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,511,0.012671999633312225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,511,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1023,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1023,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1023,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1023,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,2047,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,2047,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,2047,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,2047,0.023423999547958374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,4095,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,4095,0.04021333406368891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,4095,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,4095,0.0323786661028862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,8191,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,8191,0.06217599908510844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,8191,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,8191,0.05420800050099691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,16383,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,16383,0.10552000006039937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,16383,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,16383,0.09168533484141032
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,32767,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,32767,0.19171732664108276
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,32767,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,32767,0.1625599960486094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,3,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,3,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,3,0.012746666868527731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,3,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,7,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,7,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,15,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,31,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,31,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,31,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,63,0.012800000607967377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,63,0.012842666357755661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,63,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,63,0.01331199953953425
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,127,0.013269333789745966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,127,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,127,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,255,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,255,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,511,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,511,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,511,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,511,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1023,0.012138667205969492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1023,0.016906666258970898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1023,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,2047,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,2047,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,2047,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,2047,0.023706667125225067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,4095,0.015322666615247726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,4095,0.04154133299986521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,4095,0.015253332753976187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,4095,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,8191,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,8191,0.06274133423964183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,8191,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,8191,0.05434666574001312
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1,0.013327999661366144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,3,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,3,0.014949332922697067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,3,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,7,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,7,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,7,0.013162666310866674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,7,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,15,0.014730667074521383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,15,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,15,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,15,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,31,0.014752000570297241
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,31,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,31,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,31,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,63,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,63,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,63,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,63,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,127,0.01471466695268949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,127,0.014197333405415217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,127,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,127,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,255,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,255,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,255,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,255,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,511,0.014997333288192749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,511,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,511,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,511,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1023,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1023,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1023,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1023,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,2047,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,2047,0.04053866614898046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,2047,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,2047,0.034661332766215004
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,4095,0.01700266698996226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,4095,0.06260266900062561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,4095,0.016927999754746754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,4095,0.05592533449331919
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,8191,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,8191,0.10549867153167725
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,8191,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,8191,0.09074667096138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,3,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,3,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,3,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,3,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,7,0.014842666685581207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,7,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,7,0.01360000049074491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,7,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,15,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,15,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,15,0.013557333499193192
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,15,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,31,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,31,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,31,0.015589332828919092
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,63,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,63,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,63,0.01358933374285698
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,63,0.013749333719412485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,127,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,127,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,127,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,127,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,255,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,255,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,255,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,255,0.014815999815861383
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,511,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,511,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,511,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,511,0.017045332739750545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1023,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1023,0.023285334308942158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1023,0.01481066644191742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1023,0.02128000060717265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,2047,0.017322666943073273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,2047,0.04121066629886627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,2047,0.01738133281469345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,2047,0.03363733241955439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,4095,0.017397332936525345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,4095,0.06386133531729381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,4095,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,4095,0.05624533196290334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1,0.018543999642133713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,3,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,3,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,3,0.0174346665541331
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,3,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,7,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,7,0.018719999740521114
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,7,0.017301333447297413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,7,0.017210666090250015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,15,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,15,0.017450666675964992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,15,0.016943999876578648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,15,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,31,0.017429333180189133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,31,0.019018666197856266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,31,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,31,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,63,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,63,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,63,0.017349333812793095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,63,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,127,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,127,0.01886933296918869
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,127,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,127,0.017258666455745697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,255,0.01809599995613098
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,255,0.01735466718673706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,255,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,255,0.01692266638080279
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,511,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,511,0.023775999744733173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,511,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,511,0.022885332504908245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1023,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1023,0.03585600107908249
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1023,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1023,0.029919999341169994
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,2047,0.02117866774400075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,2047,0.058789332707722984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,2047,0.02103466788927714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,2047,0.052832002441088356
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,4095,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,4095,0.09744000434875488
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,4095,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,4095,0.08494399984677632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,3,0.010549332946538925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,7,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,63,0.011509332805871964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,127,0.011482667177915573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,255,0.011567999919255575
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,255,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,255,0.013983999689420065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,511,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,511,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1023,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1023,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1023,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,2047,0.013237333546082178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,2047,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,2047,0.013007999708255133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,4095,0.013354666531085968
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,4095,0.01706133286158244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,4095,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,4095,0.016837333639462788
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,8191,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,8191,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,8191,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,8191,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,16383,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,16383,0.01926933353145917
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,16383,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,16383,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,32767,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,32767,0.021365332106749218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,32767,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,32767,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,65535,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,65535,0.026880001028378803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,65535,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,65535,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,131071,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,131071,0.04417600234349569
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,131071,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,131071,0.03518400092919668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,7,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,7,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,7,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,7,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,15,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,15,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,31,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,63,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,127,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,255,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,255,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,511,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,511,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1023,0.010687999427318573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1023,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1023,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,2047,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,2047,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,2047,0.015493333339691162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,4095,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,4095,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,4095,0.01533866673707962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,8191,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,8191,0.017914666483799618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,8191,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,8191,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,16383,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,16383,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,16383,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,16383,0.020261333634455998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,32767,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,32767,0.029669334491093952
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,32767,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,32767,0.023578666150569916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,65535,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,65535,0.04131199916203817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,65535,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,65535,0.035301332672437034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,131071,0.015002666662136713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,131071,0.06449066599210103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,131071,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,131071,0.05620799958705902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1,0.017279999951521557
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,3,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,3,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,3,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,3,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,7,0.01724799970785777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,7,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,7,0.017370666066805523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,7,0.018474667022625606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,15,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,15,0.018954666952292126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,15,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,15,0.01701333373785019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,31,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,31,0.019199999670187633
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,31,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,31,0.017050666113694508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,63,0.019248000035683315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,63,0.018986667195955913
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,63,0.017616000026464462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,63,0.017269333203633625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,127,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,127,0.018911999960740406
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,127,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,127,0.016986666868130367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,255,0.019029332945744198
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,255,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,255,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,255,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,511,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,511,0.02499199906984965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,511,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,511,0.022837333381175995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1023,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1023,0.035616000493367515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1023,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1023,0.03133333226044973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,2047,0.021477334201335907
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,2047,0.05907199780146281
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,2047,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,2047,0.05212800204753876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1,0.025631998976071674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1,0.027104000250498455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1,0.02518933266401291
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,3,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,3,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,3,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,3,0.025381334125995636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,7,0.025749333202838898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,7,0.025199999411900837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,7,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,7,0.02518400053183238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,15,0.025759999950726826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,15,0.025936000049114227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,15,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,15,0.02531733363866806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,31,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,31,0.027562665442625683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,31,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,31,0.025077333052953083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,63,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,63,0.026421333352724712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,63,0.025349333882331848
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,63,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,127,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,127,0.02533866713444392
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,127,0.02521066615978877
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,127,0.0252960001428922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,255,0.02717866748571396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,255,0.027429332335789997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,255,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,255,0.025008000433444977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,511,0.027562665442625683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,511,0.039520000418027244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,511,0.025306666890780132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,511,0.033626665671666466
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1023,0.025797332326571148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1023,0.05825066566467285
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1023,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1023,0.052144000927607216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,2047,0.029738667110602062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,2047,0.10045866171518962
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,2047,0.02935466667016347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,2047,0.088128000497818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1,0.027664000789324444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1,0.027242665489514668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,3,0.026880001028378803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,3,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,3,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,3,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,7,0.02550400048494339
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,7,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,7,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,7,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,15,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,15,0.02716800073782603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,15,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,15,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,31,0.027456000447273254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,31,0.027466667195161183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,31,0.02508266766866048
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,63,0.02664533257484436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,63,0.02718399961789449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,63,0.025631998976071674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,63,0.025333332518736523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,127,0.027087998886903126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,127,0.02720000098148982
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,127,0.025045332809289295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,127,0.025589334468046825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,255,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,255,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,255,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,255,0.02493866781393687
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,511,0.02712533374627431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,511,0.03962666789690653
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,511,0.025978667040665943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,511,0.03565866748491923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1023,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1023,0.0583840012550354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1023,0.025861332813898723
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1023,0.052202666799227394
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1,0.04174399872620901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1,0.04174399872620901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1,0.03958400090535482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1,0.0397119993964831
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,3,0.04201599955558777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,3,0.04178666571776072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,3,0.03961600114901861
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,3,0.03938666731119156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,7,0.04153066625197729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,7,0.041536000867684685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,7,0.03952533255020777
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,7,0.0395359992980957
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,15,0.0418453315893809
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,15,0.04200000067551931
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,15,0.03933866570393244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,15,0.03947199881076813
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,31,0.04166933397452036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,31,0.04233066737651825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,31,0.04001600046952566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,31,0.039818666875362396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,63,0.04162666698296865
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,63,0.04165333261092504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,63,0.03945599993069967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,63,0.03958933303753535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,127,0.042709335684776306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,127,0.04192533095677694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,127,0.039221333960692085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,127,0.039162665605545044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,255,0.04379733403523763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,255,0.04364799956480662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,255,0.040991999208927155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,255,0.03980266551176707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,511,0.04363733530044556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,8191,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,511,0.06438933312892914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,511,0.04005333284536997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,511,0.05898666878541311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1023,0.04390400151411692
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1023,0.10149866342544556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1023,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,7,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,7,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,15,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,15,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,15,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1023,0.08941333492596944
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,63,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,63,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,127,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,255,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,511,0.011525332927703857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,511,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,511,0.012437333663304647
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1023,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1023,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1023,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,2047,0.013397333522637686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,2047,0.015216000378131866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,2047,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,2047,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,4095,0.017077332983414333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,4095,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,8191,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,8191,0.018330667167901993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,8191,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,8191,0.0169813334941864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,16383,0.01320533330241839
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,16383,0.02125866711139679
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,16383,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,16383,0.0195573332409064
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,32767,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,32767,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,32767,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,32767,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,65535,0.013610667238632837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,65535,0.041663999358812966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,65535,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,65535,0.03346133232116699
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,131071,0.015402667224407196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,131071,0.06420266628265381
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,131071,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,131071,0.05540800094604492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,3,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,3,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,3,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,7,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,15,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,31,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,31,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,63,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,127,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,511,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,511,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1023,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1023,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1023,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,2047,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,2047,0.014853333433469137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,2047,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,2047,0.01482133318980535
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,4095,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,4095,0.01695999999841054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,4095,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,4095,0.016613333175579708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,8191,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,8191,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,8191,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,8191,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,16383,0.012896000097195307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,16383,0.025242666403452556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,16383,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,16383,0.02311466634273529
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,32767,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,32767,0.040709334115187325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,32767,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,32767,0.03147733211517334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,65535,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,65535,0.06252799928188324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,65535,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,65535,0.05386666456858317
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,131071,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,131071,0.10988266269365947
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,131071,0.015295999745527903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,131071,0.09095999598503113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1023,0.012805332740147909
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1023,0.011546666423479715
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,2047,0.015247999380032221
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,2047,0.017071999609470367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,4095,0.01703466723362605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,4095,0.017103999853134155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,8191,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,8191,0.02032533288002014
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,16383,0.039290666580200195
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,16383,0.025301332275072735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,32767,0.061706667145093284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,32767,0.039887999494870506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,65535,0.10266666611035664
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,65535,0.06181866427262624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,131071,0.10610666871070862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,131071,0.1876266598701477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1,0.01250133290886879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,3,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,511,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1023,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1023,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,2047,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,2047,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,4095,0.023002666731675465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,4095,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,8191,0.035642666121323906
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,8191,0.02554133286078771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,16383,0.061621333161989846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,16383,0.039674667020638786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,32767,0.10108799735705058
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,32767,0.0610346645116806
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,65535,0.1832266648610433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,65535,0.1023520032564799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,131071,0.3494186798731486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,131071,0.18804800510406494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,255,0.011535999675591787
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,511,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,511,0.011493333925803503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1023,0.011584000041087469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1023,0.01163200040658315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,2047,0.011744000017642975
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,2047,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,4095,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,4095,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,8191,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,8191,0.015061333775520325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,16383,0.019226666539907455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,16383,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,32767,0.027136000494162243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,32767,0.027232001225153606
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,65535,0.031258667508761086
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,65535,0.029882666965325672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,131071,0.0353973334034284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,131071,0.033941333492596946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,7,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,15,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,31,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,511,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1023,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,2047,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,2047,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,4095,0.01292266696691513
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,4095,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,8191,0.015263999501864115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,8191,0.014943999548753103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,16383,0.01952533299724261
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,16383,0.020949333906173706
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,32767,0.0234400009115537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,32767,0.024688000480333965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,65535,0.02571200082699458
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,65535,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,131071,0.03297599901755651
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,131071,0.029333333174387615
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,3,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,15,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,31,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,31,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,127,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,255,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1023,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,2047,0.011792000383138657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,2047,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,4095,0.012757333616415659
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,4095,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,8191,0.017055999487638474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,8191,0.01699200024207433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,16383,0.020282667130231857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,32767,0.023013333479563396
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,32767,0.022826666633288067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,65535,0.025114665428797405
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,65535,0.025221332907676697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,131071,0.03186133255561193
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,131071,0.027509334186712902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,7,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,15,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,63,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,127,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,255,0.011765333513418833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,511,0.011519999553759893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,511,0.011685332904259363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1023,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1023,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,2047,0.012928000340859095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,2047,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,4095,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,4095,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,8191,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,8191,0.01749333366751671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,16383,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,16383,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,32767,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,32767,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,65535,0.025370667378107708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,65535,0.022997332115968067
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,131071,0.04409066836039225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,131071,0.029653333127498627
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,511,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,511,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,2047,0.018906666586796444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,2047,0.01716800034046173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,131071,0.026399999856948853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,4095,0.021146667500336964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,4095,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,8191,0.03946666667858759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,8191,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,16383,0.06038933495680491
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,16383,0.04172799984614054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,32767,0.10227732857068379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,32767,0.062128002444903054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,65535,0.18456000089645386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,65535,0.1032480001449585
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1,0.01173866664369901
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,3,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,7,0.011658667276302973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,15,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,31,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,127,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,127,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,511,0.013072000195582708
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,511,0.0122079998254776
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1023,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1023,0.013248000293970108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,2047,0.021045332153638203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,2047,0.01947733387351036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,4095,0.03923733284076055
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,4095,0.02510400116443634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,8191,0.04030933231115341
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,16383,0.10107733805974324
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,16383,0.06039466460545858
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,32767,0.18265599012374878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,32767,0.10055999954541524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,65535,0.34621334075927734
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,65535,0.18118399381637573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,7,0.010399999717871347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,31,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,255,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,255,0.011551999797423681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,511,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,511,0.011711999773979187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1023,0.012128000458081564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1023,0.011813333878914515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,2047,0.015423999478419622
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,2047,0.013951999445756277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,4095,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,4095,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,8191,0.01746133342385292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,8191,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,16383,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,16383,0.020938667158285778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,32767,0.02369600037733714
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,32767,0.02145066608985265
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,65535,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,65535,0.024746666351954143
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,131071,0.04669333497683207
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,131071,0.03166933357715607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,15,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,31,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,63,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,127,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1023,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,2047,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,4095,0.017029333859682083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,4095,0.016682667036851246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,8191,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,8191,0.01939733326435089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,16383,0.02130666623512904
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,16383,0.019920000185569126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,32767,0.02526933451493581
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,32767,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,65535,0.04113066693147024
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,65535,0.026757332185904186
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,131071,0.062458669145902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,131071,0.0435146689414978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,15,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,63,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,511,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,511,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1023,0.015146666516860327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,2047,0.021216000119845074
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,2047,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,4095,0.038549333810806274
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,4095,0.025066666305065155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,8191,0.06072533130645752
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,8191,0.03972800076007843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,16383,0.10090667009353638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,16383,0.06074133515357971
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,32767,0.18397865692774454
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,32767,0.10127466917037964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1,0.013157332936922709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,3,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,3,0.013050666699806849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,7,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,7,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,15,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,31,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,31,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,63,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,63,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,127,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,127,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,255,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,255,0.01321600005030632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,511,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,511,0.01421333352724711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1023,0.01934933289885521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1023,0.017562666287024815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,2047,0.03977066775163015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,2047,0.025701334079106648
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,4095,0.06170133252938589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,4095,0.04062933226426443
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,8191,0.10308800141016643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,8191,0.06227200229962667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,16383,0.18572799364725748
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,16383,0.10380799571673076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,32767,0.3503466844558716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,32767,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,3,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,3,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,7,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,7,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,32767,0.1865653395652771
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,15,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,15,0.012608000387748083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,31,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,31,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,63,0.013189333180586496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,63,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,127,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,127,0.013301332791646322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,255,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,511,0.015098666151364645
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,511,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1023,0.019653332730134327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1023,0.01754666616519292
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,2047,0.0402399996916453
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,2047,0.026613332331180573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,4095,0.06251733501752217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,4095,0.04181333382924398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,8191,0.10358400146166484
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,8191,0.06282133360703786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,3,0.015482666591803232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,3,0.014933332800865173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,7,0.01526933287580808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,7,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,15,0.015210667004187902
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,15,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,31,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,31,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,63,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,63,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,127,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,127,0.014789332946141561
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,255,0.015157333264748255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,255,0.014837333311637243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,511,0.01930133377512296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,511,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1023,0.03443733354409536
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1023,0.02333866556485494
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,2047,0.06055466830730438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,2047,0.04005333284536997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,4095,0.10346133510271709
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,4095,0.06247999767462412
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,8191,0.185808002948761
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,8191,0.10542933146158855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1,0.015317333241303762
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,3,0.01632533346613248
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,3,0.014906667172908783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,7,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,15,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,15,0.015024000157912573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,31,0.01657066618402799
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,31,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,63,0.01629866659641266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,63,0.014959999670584997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,127,0.01621866722901662
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,127,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,255,0.016421332955360413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,255,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,511,0.020261333634455998
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,511,0.01709866647919019
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1023,0.03419733295838038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1023,0.02295999974012375
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,2047,0.06195199986298879
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,2047,0.04146133363246918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,4095,0.10521066188812256
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,4095,0.06307733555634816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1,0.019237333287795384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,3,0.019359999646743137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,3,0.018709332992633183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,7,0.01921066641807556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,7,0.01877333347996076
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,15,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,15,0.01897066707412402
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,31,0.019391999890406925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,31,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,63,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,63,0.01882133384545644
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,127,0.020917333662509918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,127,0.01903466631968816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,255,0.019744000087181728
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,255,0.01915733392039935
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,511,0.03382933388153712
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,511,0.02347733328739802
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1023,0.05442666510740916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1023,0.03563733398914337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,2047,0.09756799538930257
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,2047,0.05977066854635874
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,4095,0.17310933272043863
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,4095,0.097461332877477
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,7,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,255,0.01071999967098236
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,511,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1023,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,2047,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,2047,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,4095,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,8191,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,8191,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,16383,0.021141332884629566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,16383,0.020997333029905956
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,32767,0.024874667326609295
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,32767,0.0225600004196167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,65535,0.02752000093460083
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,65535,0.042037333051363625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,131071,0.043749332427978516
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,131071,0.06481599807739258
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,7,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,63,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,127,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,511,0.011424000064531961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,2047,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,2047,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,4095,0.01863466699918111
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,4095,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,8191,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,8191,0.019146667172511418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,16383,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,16383,0.02109333376089732
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,32767,0.039503999054431915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,32767,0.0272533322374026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,65535,0.060746664802233376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,65535,0.04061333338419596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,131071,0.10091200470924377
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,131071,0.061119998494784035
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1,0.01894933357834816
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,3,0.02093333254257838
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,3,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,7,0.021183999876181286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,7,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,15,0.020901332298914593
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,15,0.019088000059127808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,31,0.021327999730904896
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,31,0.019167999426523846
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,63,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,63,0.019306667149066925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,127,0.020634666085243225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,127,0.018992000569899876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,255,0.019882666567961376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,255,0.018415999909241993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,511,0.03363733241955439
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,511,0.02422400067249934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1023,0.05385066568851471
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1023,0.03621866554021835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,2047,0.09703999757766724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,2047,0.05989866455396017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,3,0.029365333418051403
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,3,0.025536000728607178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,7,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,7,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,15,0.029637334247430164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,15,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,31,0.02961066613594691
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,31,0.02736533433198929
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,63,0.029504001140594482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,63,0.027061333258946735
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,127,0.02958933264017105
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,127,0.027327999472618103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,255,0.03302400062481562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,255,0.02537599951028824
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,511,0.05395199855168661
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,511,0.037765334049860634
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1023,0.09276266892751057
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1023,0.05643199880917867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,2047,0.1747573415438334
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,2047,0.09918933113416036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1,0.031199999153614044
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1,0.027674667537212372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,3,0.031343999008337654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,3,0.02773333340883255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,7,0.02938133229811986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,7,0.02731200059254964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,15,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,15,0.02717333287000656
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,31,0.031397332747777305
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,31,0.02739733209212621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,63,0.03197866678237915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,63,0.027744000156720478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,127,0.031557333966096245
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,127,0.02771199991305669
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,255,0.0335359995563825
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,255,0.027615999182065327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,511,0.05428266525268555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,511,0.03966933240493139
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1023,0.0934933324654897
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1023,0.05677866439024607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1,0.0481279989083608
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1,0.042026668787002563
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,3,0.047968000173568726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,3,0.04176533222198486
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,7,0.04790399968624115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,7,0.04165866722663244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,15,0.048138668139775596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,15,0.041834667325019836
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,31,0.04781866570313772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,31,0.04159999887148539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,63,0.048885335524876915
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,63,0.04200533529122671
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,127,0.050250664353370667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,127,0.041989331444104515
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,255,0.054469332098960876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,255,0.04384533564249674
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,511,0.0932426651318868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,511,0.06223999957243601
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1023,0.16982932885487875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1023,0.0974666674931844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,7,0.01126933346192042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,15,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,31,0.010698666175206503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,255,0.010725333044926325
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,255,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,511,0.011370666325092316
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,511,0.011296000331640244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1023,0.012282667060693106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1023,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,2047,0.017018667111794155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,2047,0.016869333883126576
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,4095,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,4095,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,8191,0.019813333948453266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,8191,0.018730666488409042
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,16383,0.02316266546646754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,16383,0.021456000705560047
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,32767,0.0417546679576238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,32767,0.027215999861558277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,65535,0.06208533545335134
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,65535,0.041562666495641075
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,131071,0.1034399966398875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,131071,0.06217599908510844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,7,0.01201066623131434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,7,0.010762666662534079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,63,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,255,0.010735999792814255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,511,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,511,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1023,0.011722666521867117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1023,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,2047,0.015173333386580149
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,2047,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,4095,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,4095,0.017477333545684814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,8191,0.023269332945346832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,8191,0.02077866718173027
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,16383,0.03921599934498469
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,16383,0.025311999022960663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,32767,0.06030400097370148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,32767,0.03989866624275843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,65535,0.10166399677594502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,65535,0.060959999759991966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,131071,0.18658665815989176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,131071,0.10632000366846721
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,3,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,3,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,3,0.01139733319481214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,7,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,7,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,15,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,15,0.011343999455372492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,31,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,31,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,63,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,127,0.011264000087976456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,255,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,255,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,511,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1023,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1023,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1023,0.012725333372751871
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,2047,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,2047,0.015168000012636185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,2047,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,4095,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,4095,0.017344000438849132
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,4095,0.013258667041858038
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,4095,0.016879999389251072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,8191,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,8191,0.01950399950146675
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,8191,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,8191,0.01911466692884763
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,16383,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,16383,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,16383,0.012901333471139273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,16383,0.023018665611743927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,32767,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,32767,0.041375999649365745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,32767,0.013370666652917862
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,32767,0.031925333042939506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,65535,0.013125333935022354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,65535,0.0628053347269694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,65535,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,65535,0.05373866856098175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,131071,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,131071,0.10994666814804077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,131071,0.015354666858911514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,131071,0.09213333328564961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,3,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,3,0.011498666057984034
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,7,0.01028266673286756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,7,0.010773333410422007
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,15,0.010138666878143946
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,15,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,15,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,31,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,31,0.010666667173306147
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,31,0.011237333218256632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,63,0.010575999816258749
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,63,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,127,0.010442666709423065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,255,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,511,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,511,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,511,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,511,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1023,0.010591999938090643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1023,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1023,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1023,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,2047,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,2047,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,2047,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,2047,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,4095,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,4095,0.012495999534924826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,4095,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,4095,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,8191,0.010703999549150467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,8191,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,8191,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,8191,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,16383,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,16383,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,16383,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,16383,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,32767,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,32767,0.02532266577084859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,32767,0.01051733394463857
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,32767,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,65535,0.010437333335479101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,65535,0.031034665803114574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,65535,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,65535,0.02959999938805898
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,131071,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,131071,0.033946665624777474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,131071,0.012975999464591345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,131071,0.03398400048414866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1,0.010714666297038397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,3,0.010512000570694605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,3,0.010469333579142889
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,7,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,15,0.010608000059922537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,31,0.010559999694426855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,63,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,63,0.010661333799362183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,127,0.010506667196750641
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,255,0.010480000327030817
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,255,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,255,0.011589333415031433
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,511,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,511,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1023,0.011973333855470022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1023,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1023,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,2047,0.011445333560307821
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,2047,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,2047,0.011402666568756104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,2047,0.012890666723251343
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,4095,0.011429333438475927
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,4095,0.012794667234023413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,4095,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,4095,0.012869333227475485
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,8191,0.011253333340088526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,8191,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,8191,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,8191,0.017221332838137943
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,16383,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,16383,0.01931200052301089
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,16383,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,16383,0.020879998803138733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,32767,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,32767,0.023141334454218548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,32767,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,32767,0.023082666099071503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,65535,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,65535,0.025498665869235992
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,65535,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,65535,0.025173333783944447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,131071,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,131071,0.029391999046007793
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,131071,0.013066666821638743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,131071,0.028970666229724884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1,0.010672000547250112
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,3,0.010496000448862711
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,3,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,7,0.011306667079528173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,15,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,15,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,31,0.011407999942700068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,63,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,127,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,255,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,511,0.01231466606259346
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,511,0.01267733300725619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1023,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1023,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,2047,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,2047,0.016842667013406754
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,2047,0.01332266628742218
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,2047,0.01646399994691213
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,4095,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,4095,0.01913600042462349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,4095,0.014869333555301031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,4095,0.019285333653291065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,8191,0.013173333058754602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,8191,0.02515200028816859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,8191,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,8191,0.023183998962243397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,16383,0.013290667285521826
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,16383,0.03979733337958654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,16383,0.01313599944114685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,16383,0.03145066648721695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,32767,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,32767,0.06283199787139893
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,32767,0.013434667140245438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,32767,0.05421866476535797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,65535,0.013424000392357508
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,65535,0.10762666662534077
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,65535,0.01441066712141037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,65535,0.0902239978313446
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,131071,0.01523200049996376
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,131071,0.19809067249298096
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,131071,0.015706667055686314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,131071,0.1637226641178131
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,15,0.010602666685978571
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,15,0.010879999647537867
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,31,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,63,0.010565333068370819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,255,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,255,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,511,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,511,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1023,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1023,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1023,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,2047,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,2047,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,2047,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,2047,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,4095,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,4095,0.014111999422311783
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,4095,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,4095,0.014762666076421738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,8191,0.013077333569526672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,8191,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,8191,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,8191,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,16383,0.012960000584522883
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,16383,0.019071999937295914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,16383,0.012784000486135483
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,16383,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,32767,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,32767,0.021231998999913532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,32767,0.012847999731699625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,32767,0.021173333128293354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,65535,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,65535,0.025258667767047882
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,65535,0.013045333325862885
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,65535,0.023365333676338196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,131071,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,131071,0.03188266605138779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,131071,0.015103999525308609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,131071,0.029866665601730347
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,3,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,3,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,7,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,15,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,31,0.011365332951148352
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,63,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,127,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,127,0.011839999506870905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,255,0.011434666812419891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,255,0.011802667131026586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,255,0.011258666714032492
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,511,0.011354666203260422
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,511,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,511,0.01219733307758967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,511,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1023,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1023,0.013199999928474426
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1023,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1023,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,2047,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,2047,0.019109333554903667
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,2047,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,2047,0.01929066702723503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,4095,0.01479999969402949
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,4095,0.025125332176685333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,4095,0.01505600040157636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,4095,0.02345066765944163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,8191,0.01293333371480306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,8191,0.0397173340121905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,8191,0.01303999995191892
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,8191,0.0316746657093366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,16383,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,16383,0.06169599791367849
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,16383,0.013429333766301474
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,16383,0.05401599903901418
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,32767,0.015226667126019796
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,32767,0.10452799995740254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,32767,0.014874666929244995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,32767,0.0909440020720164
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,65535,0.014773332824309668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,65535,0.18996266523996988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,65535,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,65535,0.16262933611869812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1,0.012831999609867731
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,3,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,3,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,3,0.01302933320403099
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,3,0.013056000073750814
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,7,0.012357333054145178
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,7,0.012789333860079447
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,7,0.012639999389648438
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,7,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,15,0.012858666479587555
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,15,0.012741333494583765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,15,0.012810666114091873
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,15,0.012736000120639801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,31,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,31,0.013061333447694778
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,31,0.012863999853531519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,31,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,63,0.013221333424250284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,63,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,63,0.01301866645614306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,63,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,127,0.012709333250919977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,127,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,127,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,127,0.012730666746695837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,255,0.012650666137536367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,255,0.011642667154471079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,255,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,255,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,511,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,511,0.014741333822409311
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,511,0.012874666601419449
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,511,0.014271999398867289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1023,0.01714133347074191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1023,0.012634667257467905
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1023,0.015418666104475657
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,2047,0.01599466676513354
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,2047,0.02535466601451238
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,2047,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,2047,0.023498666783173878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,4095,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,4095,0.04080000023047129
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,4095,0.015274666249752045
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,4095,0.03345600018898646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,8191,0.015141333142916361
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,8191,0.062352001667022705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,8191,0.015066667149464289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,8191,0.0544106662273407
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,16383,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,16383,0.1056160032749176
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,16383,0.015546667079130808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,16383,0.09092799822489421
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1,0.014757333944241205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1,0.015184000134468079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,3,0.01492799942692121
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,3,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,3,0.015119999647140503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,3,0.015471999843915304
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,7,0.015541333705186844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,7,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,7,0.015018666783968607
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,7,0.013999999811251959
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,15,0.015189333508412043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,15,0.014954666296641031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,15,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,15,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,31,0.01504533365368843
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,31,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,31,0.014533333480358124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,31,0.015072000523408255
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,63,0.015082667271296183
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,63,0.015135999768972397
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,63,0.014938666174809137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,63,0.014922666052977243
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,127,0.015034666905800501
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,127,0.015200000256299973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,127,0.014970666418472925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,127,0.01509333277742068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,255,0.015205333630243937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,255,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,255,0.014917333920796713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,255,0.01498666654030482
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,511,0.01544533297419548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,511,0.01725333308180173
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,511,0.017082666357358296
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1023,0.015125333021084467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1023,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1023,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1023,0.021354667842388153
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,2047,0.018800000349680584
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,2047,0.041493333876132965
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,2047,0.017466666797796886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,2047,0.03533866753180822
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,4095,0.01710933322707812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,4095,0.06229866544405619
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,4095,0.01729600007335345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,4095,0.05605333546797434
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,8191,0.019082666685183842
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,8191,0.10697066783905029
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,8191,0.017456000049908955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,8191,0.0921013355255127
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,3,0.010709332923094431
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,3,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,3,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,7,0.010431999961535135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,15,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,31,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,31,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,63,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,63,0.010474666953086853
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,127,0.01118933285276095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,255,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,255,0.010650667051474253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,255,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,511,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,511,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,511,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1023,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,2047,0.012682666381200155
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,2047,0.012938667088747025
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,2047,0.012879999975363413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,2047,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,4095,0.01309866706530253
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,4095,0.016976000120242436
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,4095,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,4095,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,8191,0.013317332913478216
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,8191,0.017973333597183228
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,8191,0.012954667210578918
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,8191,0.017418666432301205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,16383,0.013088000317414602
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,16383,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,16383,0.012986666212479273
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,16383,0.01964266722400983
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,32767,0.013466666142145792
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,32767,0.02242133269707362
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,32767,0.012853333105643591
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,32767,0.02139200021823247
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,65535,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,65535,0.027493332823117573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,65535,0.013194666554530462
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,65535,0.025519999365011852
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,131071,0.015029333531856537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,131071,0.04334400097529093
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,131071,0.014912000546852747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,131071,0.03391999999682108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1,0.018863999595244724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1,0.01893866683046023
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1,0.01798933371901512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1,0.017909333109855652
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,3,0.018960000326236088
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,3,0.019061333189407986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,3,0.017130666722853977
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,7,0.01899733394384384
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,3,0.01711999997496605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,7,0.017594666530688603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,7,0.01748266691962878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,7,0.017125333348910015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,15,0.018965333700180054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,15,0.01720000058412552
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,15,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,15,0.017231999586025875
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,31,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,31,0.017573333034912746
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,31,0.017157333592573803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,31,0.017194667210181553
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,63,0.01727466657757759
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,63,0.017360000560681026
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,63,0.01695466662446658
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,63,0.017317333569129307
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,127,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,127,0.01912533367673556
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,127,0.016938666502634685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,127,0.01858666663368543
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,255,0.01823466643691063
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,255,0.018816000471512478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,255,0.01736533393462499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,255,0.017605333278576534
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,511,0.019050666441520054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,511,0.0233599990606308
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,511,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,511,0.021498667697111767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1023,0.01883200059334437
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1023,0.0359946663180987
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1023,0.018981333822011948
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1023,0.029690665503342945
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,2047,0.021695998807748158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,2047,0.06037333110968272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,2047,0.021509334444999695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,2047,0.053946668903032936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,4095,0.021333334346612293
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,4095,0.09779733419418335
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,4095,0.02128533273935318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,4095,0.0848586658636729
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1,0.02555199960867564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1,0.027621333797772724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1,0.025263999899228413
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1,0.025685332715511322
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,3,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,3,0.02719466636578242
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,3,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,3,0.025231999655564625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,7,0.02553066611289978
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,7,0.02552533398071925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,7,0.025216000775496166
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,7,0.02536533276240031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,15,0.027274665733178455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,15,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,15,0.02516799916823705
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,15,0.025386666258176167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,31,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,31,0.025120000044504803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,31,0.025226667523384094
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,31,0.025829332570234936
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,63,0.025445332129796345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,63,0.025637333591779072
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,63,0.02502399931351344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,63,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,127,0.025583999852339428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,127,0.025605333348115284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,127,0.025392000873883564
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,127,0.025248001019159954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,255,0.027503999571005504
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,255,0.025514667232831318
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,255,0.02586666742960612
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,255,0.025274666647116344
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,511,0.02740799884001414
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,511,0.04029866556326548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,511,0.02611200014750163
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,511,0.03563733398914337
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1023,0.027295999228954315
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1023,0.05831466615200043
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1023,0.025802666942278545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1023,0.05256533126036326
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,2047,0.029882666965325672
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,2047,0.10087999701499939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,2047,0.029733332494894665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1,0.010618666807810465
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,3,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,3,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,7,0.010645333677530289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,2047,0.08911466598510742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,15,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,15,0.010741333166758219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,31,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,31,0.010757333288590113
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,63,0.01108266661564509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,127,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,255,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,255,0.012304000556468964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,511,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,511,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,511,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1023,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1023,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,2047,0.012885333349307379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,2047,0.015306666493415833
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,2047,0.0129120002190272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,2047,0.01540800059835116
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,4095,0.01310933381319046
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,4095,0.017338667064905167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,4095,0.013104000439246496
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,4095,0.017184000462293625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,8191,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,8191,0.018383999665578205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,8191,0.012906666845083237
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,8191,0.01730666682124138
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,16383,0.012991999586423239
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,16383,0.021397332350413006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,16383,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,16383,0.021136000752449036
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,32767,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,32767,0.02770666778087616
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,32767,0.012965332716703415
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,65535,0.013093333691358566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,65535,0.04164800047874451
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,65535,0.013178666432698568
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,65535,0.033514666060606636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,8191,0.060122668743133545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,131071,0.015626666446526844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,131071,0.0625546673933665
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,131071,0.015301333119471868
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1,0.011328000575304031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,131071,0.05444799860318502
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,15,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,31,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,511,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1023,0.012997332960367203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1023,0.012768000364303589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,2047,0.01708799973130226
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,2047,0.016949333250522614
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,4095,0.017093333105246227
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,4095,0.01722666621208191
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,8191,0.02186666677395503
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,8191,0.021274665991465252
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,16383,0.03963200002908707
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,16383,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,32767,0.06170133252938589
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,32767,0.03782399992148081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,65535,0.101583997408549
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,65535,0.06168533364931742
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,131071,0.189082662264506
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1,0.011834666132926941
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,131071,0.10526933272679646
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,3,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,15,0.010389333590865135
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,511,0.010837333897749582
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,511,0.01101333275437355
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1023,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1023,0.010794666906197866
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,2047,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,2047,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,4095,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,4095,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,8191,0.015178666760524115
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,8191,0.01655999943614006
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,16383,0.01926400015751521
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,16383,0.021066665649414062
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,32767,0.029322666426499683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,32767,0.029050665597120922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,65535,0.03366400053103765
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,65535,0.03050133337577184
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,131071,0.03188266605138779
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,31,0.010522666076819101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,31,0.010634666929642359
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,63,0.01055466632048289
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,255,0.010533332824707031
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,511,0.01097600037852923
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1023,0.011087999989589056
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,2047,0.012752000242471695
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,2047,0.012826666235923767
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,4095,0.012981332838535309
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,4095,0.012837332983811697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,8191,0.01732800031701724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,8191,0.017024000485738117
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,16383,0.021194666624069214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,16383,0.02081599955757459
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,32767,0.025546667476495106
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,32767,0.024458666642506916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,65535,0.027461332579453785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,65535,0.025290665527184803
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,131071,0.030026666820049286
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,131071,0.02733866622050603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1,0.010933333386977514
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,7,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,511,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1023,0.013541333377361298
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1023,0.013386666774749756
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,2047,0.018933333456516266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,2047,0.01717866708834966
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,4095,0.02160000056028366
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,4095,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,8191,0.03955200066169103
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,8191,0.02606400102376938
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,16383,0.06156266729036967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,16383,0.037776000797748566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,32767,0.10338133573532104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,32767,0.061759998401006065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,65535,0.18548800547917685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,65535,0.1034346620241801
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,131071,0.35283732414245605
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,131071,0.18879467248916626
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,3,0.01166933278242747
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,31,0.010682666053374609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,63,0.010656000425418219
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,255,0.010693332801262537
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1023,0.010954666882753372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,2047,0.013888000200192133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,2047,0.01328533391157786
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,4095,0.014794666320085526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,4095,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,8191,0.019343999524911244
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,8191,0.019002666076024372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,16383,0.022634667654832203
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,16383,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,32767,0.025034666061401367
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,32767,0.02309866746266683
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,65535,0.027493332823117573
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,65535,0.0232640008131663
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,131071,0.04708800216515859
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,131071,0.031301334500312805
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1,0.011301333705584208
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1,0.012917333592971167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,3,0.011333333949247995
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,15,0.011727999895811081
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,511,0.012949333836634954
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,511,0.012944000462690989
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1023,0.014965333044528961
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1023,0.014250667144854864
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,2047,0.02317333221435547
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,2047,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,4095,0.03941333293914795
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,4095,0.025466665625572205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,8191,0.06202666461467743
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,8191,0.039664000272750854
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,16383,0.10127466917037964
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,16383,0.060122668743133545
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,32767,0.1837760011355082
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,32767,0.1009386678536733
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,65535,0.3477333386739095
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,65535,0.18106667200724283
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1,0.013082666943470636
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1,0.012762666990359625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,3,0.013167999684810638
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,3,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,7,0.013183999806642532
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,7,0.012970666090647379
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,15,0.013034666577974955
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,15,0.013002666334311167
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,31,0.013114667187134424
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,31,0.013141332815090815
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,63,0.013130666067202887
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,63,0.012778667112191519
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,127,0.013151999562978745
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,127,0.013013333082199097
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,255,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,255,0.01314666618903478
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,511,0.015194666882356008
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,511,0.015040000279744467
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1023,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1023,0.017237332959969837
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,2047,0.03769599894682566
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,2047,0.02741333345572154
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,4095,0.062261333068211876
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,4095,0.040048000713189445
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,8191,0.10356266299883525
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,8191,0.06264000137646993
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,16383,0.18585066000620523
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1,0.015397333850463232
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1,0.014991999914248785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,16383,0.10524266958236694
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,3,0.01532799998919169
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,3,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,7,0.015552000453074774
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,7,0.015050667027632395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,15,0.015498666713635126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,15,0.014901333798964819
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,31,0.01543466622630755
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,31,0.014975999792416891
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,63,0.01525866612792015
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,63,0.015013333410024643
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,127,0.015311999867359797
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,127,0.01516266663869222
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,255,0.01553600033124288
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,255,0.014864000181357065
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,511,0.021087999145189922
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,511,0.01747200017174085
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1023,0.03489066660404205
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1023,0.023530667026837666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,2047,0.06181866427262624
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,2047,0.04142933338880539
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,4095,0.10425066947937012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,4095,0.06377600133419037
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,8191,0.10629333058993022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,8191,0.18754132588704428
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1,0.011600000162919363
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,3,0.011312000453472137
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,15,0.011440000186363855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,63,0.011381333072980246
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,63,0.010629333555698395
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,255,0.010784000158309937
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,2047,0.013264000415802002
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,2047,0.014981333166360855
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,4095,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,4095,0.017173333714405697
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,8191,0.019253333409627277
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,8191,0.019130667050679524
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,16383,0.021157334248224895
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,16383,0.0194560003777345
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,32767,0.026506667335828144
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,32767,0.021407999098300934
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,65535,0.04192000130812327
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,65535,0.02722666660944621
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,131071,0.044122666120529175
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,131071,0.06417066852251689
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1,0.01945066700379054
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1,0.021242665747801464
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,3,0.019413333386182785
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,3,0.018917333334684372
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,7,0.019194666296243668
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,7,0.019205333044131596
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,15,0.019424000134070713
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,15,0.018895999838908512
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,31,0.019381333142518997
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,31,0.019141333798567455
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,63,0.02096533278624217
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,63,0.01904533306757609
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,127,0.021029333273569744
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,127,0.01922133316596349
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,255,0.02107200026512146
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,255,0.01916266605257988
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,511,0.03374933451414108
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,511,0.023210667073726654
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1023,0.0537120004494985
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1023,0.035418666899204254
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,2047,0.09888000289599101
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,2047,0.05912533402442932
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,4095,0.17537599802017212
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,4095,0.09798399607340495
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1,0.029989334444204967
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1,0.025834667185942333
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,3,0.029306667546431225
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,3,0.025781333446502686
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,7,0.02942399928967158
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,7,0.025477332373460133
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,15,0.029301332930723827
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,15,0.025610665480295818
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,31,0.029264000554879505
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,31,0.027269333600997925
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,63,0.029765332738558452
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,63,0.027637332677841187
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,127,0.029680001238981884
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,127,0.02769600103298823
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,255,0.033743999898433685
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,255,0.02548266698916753
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,511,0.05459733307361603
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,511,0.038831998904546104
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1023,0.09327466289202373
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,2047,0.17575999101003012
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1023,0.0569653312365214
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,2047,0.09948266545931499
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,7,0.010746666540702185
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,511,0.011007999380429586
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,2047,0.01578666642308235
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,4095,0.017642666896184284
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,4095,0.017386666188637417
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,8191,0.021018666525681812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,8191,0.01918399954835574
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,16383,0.025285333395004272
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,16383,0.02124800036350886
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,32767,0.039994666973749794
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,32767,0.02714666724205017
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,65535,0.062218666076660156
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,65535,0.04181333382924398
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,131071,0.10337066650390625
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,131071,0.06423999865849812
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,32767,0.025194667279720306
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,131071,0.03711466739575068
TRTLLM,1.2.0rc5,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,15,0.010879999647537867
