framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,64,0.008089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,32,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,16,0.006963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,8,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,128,0.011980800330638886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,4,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,2,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,1,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,128,0.01454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,32,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,64,0.010649599879980088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,16,0.006143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,4,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,8,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,2,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,1,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,64,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,128,0.014643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,32,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,16,0.006143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,4,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,1,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,2,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,128,0.014643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,64,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,16,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,32,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,4,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,2,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,1,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,128,0.015052799880504609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,32,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,64,0.010649599879980088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,8,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,16,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,2,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,4,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,1,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,128,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,64,0.01085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,16,0.006348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,32,0.007680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,8,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,4,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,2,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,1,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,128,0.015872000157833098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,64,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,32,0.010342399775981902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,16,0.007475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,8,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,1,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,4,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,128,0.017100800573825837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,64,0.011468800157308579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,32,0.010649599879980088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,8,0.006348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,16,0.007577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,4,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,2,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,128,0.022835199534893037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,64,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,32,0.013312000036239623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,16,0.009216000139713288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,8,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,4,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,1,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,128,0.023654399812221526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,64,0.016179199516773223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,16,0.009216000139713288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,8,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,4,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,32,0.013516800105571746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,2,0.006553599983453751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,1,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,128,0.03041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,64,0.017407999932765962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,32,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,16,0.010649599879980088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,8,0.007577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,2,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,1,0.006963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,128,0.0506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,4,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,64,0.022732800245285033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,32,0.015872000157833098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,16,0.011673600226640702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,8,0.008089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,2,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,4,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,128,0.05355520248413086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,64,0.023449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,32,0.015974399447441102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,16,0.011878400295972823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,4,0.006963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,8,0.009318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,2,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,1,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,32,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,64,0.02396160066127777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,8,0.010751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,16,0.011878400295972823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,4,0.007475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,2,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,128,0.05826560258865356
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,1,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,64,0.03430399894714355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,128,0.0894976019859314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,32,0.019251200556755065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,16,0.012185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,8,0.012083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,1,0.006143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,4,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,2,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,64,0.03819519877433777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,128,0.09379839897155762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,32,0.01935359984636307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,4,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,16,0.012185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,8,0.013516800105571746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,2,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,1,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,64,0.048332801461219786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,128,0.10362880229949951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,32,0.023654399812221526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,16,0.016281600296497344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,8,0.016281600296497344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,4,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,2,0.007680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,128,0.14960639476776122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,64,0.06901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,32,0.034815999865531924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,16,0.01945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,8,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,4,0.008703999966382981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,2,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,1,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,64,0.08734719753265381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,128,0.19179519414901733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,32,0.045158401131629944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,16,0.023756800591945647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,8,0.017407999932765962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,4,0.011878400295972823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,2,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,1,0.006348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,128,0.2861056089401245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,32,0.06594560146331788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,64,0.12421120405197143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,16,0.03338240087032318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,8,0.02242559939622879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,4,0.013926400244235993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,1,0.006758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,128,0.3749887943267822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,64,0.1614848017692566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,2,0.009625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,32,0.08488960266113281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,16,0.04352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,8,0.027647998929023743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,4,0.017612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,2,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,1,0.008089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,128,0.5576704025268555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,32,0.12277760505676269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,16,0.06512640118598938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,4,0.022732800245285033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,8,0.040243199467658995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,2,0.013823999464511872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,1,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,128,0.7392255783081054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,64,0.23797760009765626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,64,0.31027200222015383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,32,0.1602560043334961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,16,0.08550400137901307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,8,0.052121597528457644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,4,0.027750399708747864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,2,0.017817600071430205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,1,0.01157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,64,0.4597760200500488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,32,0.23592960834503174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,16,0.1230847954750061
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,8,0.06399999856948853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,4,0.03317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,128,0.9156607627868653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,2,0.01955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,1,0.011878400295972823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,32,0.3096575975418091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,64,0.6139904022216797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,128,1.2142592430114747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,16,0.15912959575653077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,8,0.08273919820785522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,2,0.02457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,1,0.020070399343967437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,4,0.0419840008020401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,128,0.011673600226640702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,64,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,32,0.007475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,2,0.006143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,16,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,4,0.006348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,1,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,64,0.009728000313043595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,128,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,32,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,8,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,16,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,8,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,4,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,2,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,1,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,128,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,32,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,8,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,64,0.009830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,16,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,4,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,2,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,1,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,128,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,64,0.009625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,16,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,4,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,2,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,8,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,32,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,128,0.0130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,64,0.009523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,32,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,16,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,4,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,8,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,64,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,32,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,16,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,128,0.013209599256515502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,4,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,2,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,8,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,128,0.013414399325847625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,32,0.007577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,8,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,16,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,4,0.008089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,64,0.010956799983978272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,128,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,64,0.010444799810647965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,32,0.007782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,8,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,4,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,16,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,2,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,128,0.020684799551963805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,64,0.012902399897575379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,16,0.007577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,8,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,4,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,128,0.025190401077270507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,32,0.012800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,64,0.01454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,32,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,16,0.007577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,8,0.007680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,4,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,128,0.032153600454330446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,32,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,16,0.007884799689054488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,64,0.014745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,4,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,8,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,1,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,128,0.03758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,64,0.02088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,32,0.0130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,4,0.007475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,8,0.007475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,16,0.008703999966382981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,1,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,128,0.03983359932899475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,64,0.02088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,32,0.013107199966907502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,8,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,4,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,16,0.010035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,2,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,128,0.04618239998817444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,64,0.021401600539684297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,32,0.013209599256515502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,16,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,1,0.007475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,4,0.008703999966382981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,8,0.008806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,2,0.008806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,64,0.02805759906768799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,128,0.05304319858551025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,16,0.009523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,4,0.008806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,8,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,1,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,2,0.008703999966382981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,64,0.0347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,128,0.06000639796257019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,32,0.013414399325847625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,32,0.015052799880504609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,8,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,16,0.010035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,2,0.008703999966382981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,4,0.008806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,1,0.007168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,32,0.02109439969062805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,64,0.042905598878860474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,16,0.014643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,8,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,128,0.07137280106544494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,2,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,4,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,1,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,128,0.10229760408401489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,64,0.055193597078323366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,16,0.015769599378108977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,32,0.027955201268196107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,4,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,2,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,1,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,128,0.12851200103759766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,8,0.009728000313043595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,64,0.06809599995613098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,32,0.040038400888442995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,16,0.020275199413299562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,8,0.01454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,4,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,2,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,1,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,128,0.18534400463104247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,32,0.052019202709198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,8,0.014745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,16,0.026316800713539125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,4,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,2,0.008806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,64,0.0946175992488861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,1,0.007372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,128,0.24227840900421144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,64,0.121343994140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,16,0.03932160139083862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,32,0.06410239934921265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,4,0.013209599256515502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,8,0.02109439969062805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,1,0.007680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,128,0.35399680137634276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,64,0.18196480274200438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,32,0.09379839897155762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,16,0.05263360142707825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,2,0.008908800035715102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,8,0.02181120067834854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,4,0.013414399325847625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,2,0.010035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,1,0.008499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,128,0.4583424091339111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,64,0.23930881023406983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,16,0.06533120274543762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,8,0.03870719969272614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,2,0.015052799880504609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,4,0.021401600539684297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,1,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,128,0.7051263809204101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,32,0.118886399269104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,64,0.3521536111831665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,32,0.17541120052337647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,16,0.0936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,8,0.05027840137481689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,2,0.015872000157833098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,4,0.021913599967956544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,1,0.010649599879980088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,64,0.47912960052490233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,32,0.23541760444641113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,16,0.12339199781417846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,8,0.06399999856948853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,4,0.03829759955406189
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,2,0.021196800470352172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,128,0.9282560348510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,1,0.013107199966907502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,128,1.3841407775878907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,64,0.6912000179290771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,16,0.1793023943901062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,8,0.09082880020141601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,32,0.3601408004760742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,4,0.051097601652145386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,1,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,64,0.9136128425598145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,128,1.8364416122436524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,32,0.4555776119232178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,16,0.2381824016571045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,2,0.022118400037288665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,8,0.11909120082855225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,4,0.06502400040626526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,2,0.03891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,1,0.01955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,32,0.5732351779937744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,64,1.1423744201660155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,128,2.2968320846557617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,8,0.14673919677734376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,4,0.07608320116996765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,16,0.29091839790344237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,2,0.042803201079368594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,1,0.020172800123691558
